diff --git a/.dockerignore b/.dockerignore index 6dfbea7..c20e2d8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,7 @@ venv/ **/__pycache__/ *.pyc +*.egg-info/ # Git .git @@ -13,20 +14,38 @@ node_modules src/website/node_modules # Local caches and data +.env +**/.env .cache .hypothesis .coverage secrets -# Project data directories +# Project data directories (and database dumps written by the backup service) +backups/ Data .data data/ +# Build and test outputs. The e2e harness runs on the host; e2e/runs alone +# used to add ~4 GB to every image. +build/ +dist/ +e2e/runs/ +e2e/.cache/ +e2e/.venv/ +e2e/.browser-venv/ +**/.pytest_cache +**/.ruff_cache + # Docker files Dockerfile* docker-compose*.yml +# Development-only scripts (e.g. the guarded research-study seeder) must never +# be baked into the server image. +scripts/dev/ + # OS .DS_Store diff --git a/.env b/.env deleted file mode 100644 index d10a0df..0000000 --- a/.env +++ /dev/null @@ -1,91 +0,0 @@ -# ----------------------- -# General & Server Settings -# ----------------------- -DATA_DIR=./data - -SERVER_VERSION_ID=1 -SERVER_HOST=0.0.0.0 -SERVER_PORT=8008 - -# ----------------------- -# Authentication & Rate Limits -# ----------------------- -AUTHENTICATION_TOKEN_EXPIRES_IN_SECONDS=86400 -SESSION_TOKEN_EXPIRES_IN_SECONDS=7200 -EMAIL_VERIFICATION_TOKEN_EXPIRES_IN_SECONDS=86400 -RESET_PASSWORD_TOKEN_EXPIRES_IN_SECONDS=900 -TOKEN_HOOK_ACTIVATION_IN_SECONDS=10 -DEFAULT_MAX_REQUEST_RATE_PER_HOUR=500 -MAX_REQUEST_RATE_PER_HOUR_CONFIG='{"/api/completion/multi_file_context/update":300,"/api/completion/feedback":600,"/api/completion/get":100,"/api/completion/request":600,"/api/project/activate":100,"/api/project/create":100,"/api/session/acquire":100,"/api/user/authenticate":100,"/api/user/create":100,"/api/user/delete":100,"/api/user/update":100,"/api/ws/completion":1200,"/api/ws/completion/multi_file_context":300}' - - -# ----------------------- -# Database Configuration -# ----------------------- -DB_HOST=localhost -DB_PORT=5432 -DB_USER=postgres -DB_PASSWORD=postgres -DB_NAME=code4meV2 - -# ----------------------- -# SQLAlchemy Engine Connection Config -# ----------------------- -DB_POOL_SIZE=20 -DB_MAX_OVERFLOW=30 -DB_POOL_TIMEOUT=5 -DB_POOL_RECYCLE=1800 - -# ----------------------- -# pgAdmin Configuration -# ----------------------- -PGADMIN_HOST=localhost -PGADMIN_PORT=8080 -PGADMIN_DEFAULT_EMAIL=admin@admin.com -PGADMIN_DEFAULT_PASSWORD=admin - -# ----------------------- -# Frontend Configuration -# ----------------------- -WEBSITE_HOST=http://localhost -WEBSITE_PORT=3000 -REACT_APP_GOOGLE_CLIENT_ID=909086079826-skaqesnvhn16q0vq0vn9iot3no8g67h5.apps.googleusercontent.com - -# ----------------------- -# Redis & Celery Configuration -# ----------------------- -REDIS_HOST=localhost -REDIS_PORT=6379 -CELERY_BROKER_HOST=localhost -CELERY_BROKER_PORT=6380 - -# ----------------------- -# Email Configuration -# ----------------------- -EMAIL_HOST=smtp.example.com -EMAIL_PORT=587 -EMAIL_USERNAME=YOUR_EMAIL_USERNAME -EMAIL_PASSWORD=YOUR_EMAIL_PASSWORD -EMAIL_USE_TLS=true -EMAIL_FROM=YOUR_EMAIL -VERIFICATION_URL=http://localhost:8008/api/user/verify -RESET_PASSWORD_URL=http://localhost:8008/api/user/reset-password - -# ----------------------- -# Model Configuration -# ----------------------- -PRELOAD_MODELS=false -MODEL_CACHE_DIR=./.cache -MODEL_USE_CACHE=true -MODEL_USE_COMPILE=true -MODEL_WARMUP=true - -# ----------------------- -# Thread Pool Configuration -# ----------------------- -THREAD_POOL_MAX_WORKERS=2 - -# ----------------------- -# Hugging Face Configuration -HF_TOKEN=your_huggingface_token_here -# ----------------------- \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5f635ed --- /dev/null +++ b/.env.example @@ -0,0 +1,179 @@ +# Example environment variables for the agent subsystem. +# +# PLACEHOLDER VALUES ONLY — never commit a real credential to this file. +# Copy the entries you need into your own `.env` (which is gitignored for real +# values) and fill them in there. +# +# This file documents only the agent-specific variables added by the agent +# subsystem. See README.md for the base platform's configuration (database, +# Redis, Celery, email, model serving). + + +# ─── Upstream LLM provider ─────────────────────────────────────────────────── +# +# Per merge decision 6 there is no hardcoded provider. Each `agent_profile` row +# names its own `base_url` + `api_key_ref` + `model`, and the only requirement +# is that the endpoint speaks the OpenAI-compatible chat-completions wire +# format. Ollama, Groq, OpenRouter, vLLM and OpenAI all qualify. +# +# Keys are NEVER stored in the database. A profile stores `api_key_ref`, the +# *name* of an environment variable, and the key is read from the environment +# at request time — so dumping the profile table cannot leak a credential. + +# Fallback upstream, used when a profile leaves `base_url` empty. +# Defaults to a local Ollama instance, which needs no key and costs nothing. +AGENT_UPSTREAM_BASE_URL=http://localhost:11434/v1 + +# Fallback API key, used when a profile leaves `api_key_ref` empty. +# Safe to leave blank for local providers that don't authenticate. +AGENT_UPSTREAM_API_KEY= + +# Per-provider keys. Create one variable per provider you use and point the +# matching profile's `api_key_ref` at its name. The seeded default profiles +# reference OLLAMA_API_KEY, AGENT_UPSTREAM_API_KEY and OPENAI_API_KEY. +OLLAMA_API_KEY= +OPENAI_API_KEY=your_openai_api_key_here +GROQ_API_KEY=your_groq_api_key_here +OPENROUTER_API_KEY=your_openrouter_api_key_here + + +# ─── Codex Responses API ───────────────────────────────────────────────────── +# +# Codex speaks OpenAI's proprietary Responses API rather than chat-completions, +# so it is handled as a special-cased normalization path. When a Codex profile +# targets a non-OpenAI base_url, the proprietary fields are rewritten; when it +# targets OpenAI, the payload is forwarded largely as-is. + +# Optional operator override for reasoning effort, e.g. to cap cost. +# Valid values: low | medium | high. Leave blank to let Codex decide (with a +# clamp applied for models that reject its default). +AGENT_REASONING_EFFORT= + +# Optional positive output-token cap for Codex Responses requests routed to a +# non-OpenAI provider. Leave blank to preserve the agent's requested budget. +AGENT_RESPONSES_MAX_OUTPUT_TOKENS= + +# ─── Research inference gateway and participant budgets ────────────────────── +# +# Goose and built-in study arms spend from the study's provider connection +# (the env var named by its `secret_ref`) through a metered relay. Every call +# reserves its worst-case cost before it is forwarded and settles the actual +# usage afterwards. Defaults are conservative; all values are optional. + +# Global per-call output cap (tokens); the injected max_tokens never exceeds it. +INFERENCE_OUTPUT_TOKEN_CEILING=8192 +# Below this affordable output cap a call is refused (402) instead of sent. +INFERENCE_MIN_OUTPUT_TOKENS=256 +# Multiplier on the byte-based prompt estimate (>= 1.0). +INFERENCE_INPUT_ESTIMATE_SAFETY=1.5 +# Estimator constants. +INFERENCE_INPUT_BYTES_PER_TOKEN=3 +INFERENCE_PER_MESSAGE_TOKEN_OVERHEAD=4 +# A hold older than this is charged in full by the next reservation (seconds). +INFERENCE_RESERVATION_DEADLINE_SECONDS=900 +# Wall-clock cap on one streamed response (seconds; keep below the deadline +# minus the 120 s upstream timeout). +INFERENCE_STREAM_TOTAL_TIMEOUT_SECONDS=600 +# Lifetime of the inference capability handed to Goose (seconds). Goose reads +# it once per process; liveness, budget and revocation are re-checked per call. +INFERENCE_CAPABILITY_TTL_SECONDS=604800 +# The metered relay paths get a per-IP floor of 20000 requests/hour unless +# MAX_REQUEST_RATE_PER_HOUR_CONFIG names them, e.g. +# MAX_REQUEST_RATE_PER_HOUR_CONFIG={"/api/research/inference/v1/chat/completions": 50000} + + +# ─── Production inventory (docker-compose.prod.yml) ────────────────────────── +# +# The application reads `.env` through `env_file`; the compose file additionally +# asserts the secrets below with `${VAR:?}` so `docker compose up` fails fast +# instead of a participant discovering a dead research plane. Everything the +# backend requires (no default in Code4meV2Config) is listed; values here are +# placeholders. + +# --- Secrets the research plane cannot run without -------------------------- +# HMAC key for bootstrap manifests and session/inference capabilities. Read at +# request time; a missing/blank value makes every bootstrap answer 503 and the +# server refuses to start (except under TEST_MODE). Generate with: +# python3 -c 'import secrets; print(secrets.token_urlsafe(48))' +BOOTSTRAP_SIGNING_SECRET=replace-with-a-long-random-secret +# Redis `requirepass` (auth tokens, sessions, ACP grants live there). +REDIS_PASSWORD=replace-with-a-long-random-password +# The https:// origin participants use; becomes CORS_ALLOWED_ORIGINS and the +# website's compiled backend host (the same origin, nginx proxies /api/). +PUBLIC_ORIGIN=https://study.example.org + +# --- Database / cache / broker (compose overrides the hosts and ports) ------- +DB_NAME=code4meV2 +DB_USER=postgres +DB_PASSWORD=replace-me +DB_POOL_SIZE=10 +DB_MAX_OVERFLOW=20 +DB_POOL_TIMEOUT=30 +DB_POOL_RECYCLE=1800 +# Host port Postgres is bound to on 127.0.0.1 (backups, psql); never public. +DB_HOST_PORT=5432 +# Only used outside compose (the compose file sets the container hosts/ports). +DB_HOST=localhost +DB_PORT=5432 +REDIS_HOST=localhost +REDIS_PORT=6379 +CELERY_BROKER_HOST=localhost +CELERY_BROKER_PORT=6379 + +# --- Server ------------------------------------------------------------------ +SERVER_HOST=0.0.0.0 +SERVER_PORT=8008 +SERVER_VERSION_ID=1 +# Uvicorn workers; assignment/session/receipt authority lives in Postgres. +CODE4ME_WEB_WORKERS=2 +# Comma-separated proxies whose X-Forwarded-For is trusted (rate limiting and +# uvicorn); the compose sets "*" because only nginx can reach the backend. +FORWARDED_ALLOW_IPS=127.0.0.1 +# Secure cookies (set by the compose; keep false for plain-HTTP local stacks). +COOKIE_SECURE=false +# Grace after a session ended during which its late events are still stored. +TELEMETRY_LATE_EVENT_GRACE_SECONDS=900 +# Serve the classic completion/chat rows (provider-backed by default). +CLASSIC_MODELS_ENABLED=true +PRELOAD_MODELS=false + +# --- Tokens / rate limits ---------------------------------------------------- +AUTHENTICATION_TOKEN_EXPIRES_IN_SECONDS=86400 +SESSION_TOKEN_EXPIRES_IN_SECONDS=86400 +EMAIL_VERIFICATION_TOKEN_EXPIRES_IN_SECONDS=86400 +RESET_PASSWORD_TOKEN_EXPIRES_IN_SECONDS=900 +TOKEN_HOOK_ACTIVATION_IN_SECONDS=60 +# Per client (real address behind nginx) per path per hour. Research-plane +# paths (sessions, telemetry batches, bootstrap) and the inference relay get a +# 20000/h floor regardless; override a path with MAX_REQUEST_RATE_PER_HOUR_CONFIG. +DEFAULT_MAX_REQUEST_RATE_PER_HOUR=1000 +MAX_REQUEST_RATE_PER_HOUR_CONFIG={} + +# --- Email (verification / password reset) ----------------------------------- +EMAIL_HOST=smtp.example.org +EMAIL_PORT=587 +EMAIL_USERNAME=noreply@example.org +EMAIL_PASSWORD=replace-me +EMAIL_USE_TLS=true +EMAIL_FROM=noreply@example.org +VERIFICATION_URL=https://study.example.org/verify +RESET_PASSWORD_URL=https://study.example.org/reset-password + +# --- Website / misc required by the configuration model ---------------------- +WEBSITE_HOST=website +WEBSITE_PORT=80 +# Required by the config model even when pgAdmin is not deployed. +PGADMIN_HOST=localhost +PGADMIN_PORT=5050 +PGADMIN_DEFAULT_EMAIL=admin@example.org +PGADMIN_DEFAULT_PASSWORD=replace-me + +# --- Backups (backup service) ------------------------------------------------ +BACKUP_DIR=./backups +BACKUP_RETENTION_DAYS=14 +BACKUP_INTERVAL_SECONDS=86400 +# Optional off-host copy, run with the dump path as $1, e.g. +# rclone copy "$1" remote:code4me-backups +BACKUP_OFFHOST_CMD= +# Host port for the website/nginx container (80 behind an external TLS edge). +HTTP_PORT=80 diff --git a/.github/workflows/build-managed-runtime.yml b/.github/workflows/build-managed-runtime.yml new file mode 100644 index 0000000..4e7c335 --- /dev/null +++ b/.github/workflows/build-managed-runtime.yml @@ -0,0 +1,299 @@ +name: Build managed runtime + +on: + workflow_dispatch: + inputs: + version: + description: Runtime version to stamp into the manifest + required: true + default: 0.0.1 + publishRelease: + description: Publish immutable runtime-v release assets + type: boolean + required: true + default: false + sign: + description: Code-sign the archives (macOS also notarized); needs the signing secrets + type: boolean + required: true + default: false + minPluginVersion: + description: Minimum compatible Code4Me plugin version (leave blank if unchanged) + required: false + default: "" + push: + paths: + - .github/workflows/build-managed-runtime.yml + - src/code4me2_agent/** + - packaging/** + - src/research/study/agents/** + - tests/test_participant_release.py + - pyproject.toml + - tests/code4me2_agent/** + pull_request: + paths: + - .github/workflows/build-managed-runtime.yml + - src/code4me2_agent/** + - packaging/** + - src/research/study/agents/** + - tests/test_participant_release.py + - pyproject.toml + - tests/code4me2_agent/** + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: + - os: macos-15 + platform: macos-arm64 + executable: dist/code4me2-agent/code4me2-agent + - os: macos-15-intel + platform: macos-x64 + executable: dist/code4me2-agent/code4me2-agent + - os: windows-2022 + platform: windows-x64 + executable: dist/code4me2-agent/code4me2-agent.exe + - os: ubuntu-24.04 + platform: linux-x64 + executable: dist/code4me2-agent/code4me2-agent + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: | + packaging/requirements-runtime.lock + packaging/requirements-build.lock + packaging/requirements-test.lock + - name: Verify native runner architecture + shell: bash + env: + EXPECTED_PLATFORM: ${{ matrix.platform }} + run: | + python - <<'PY' + import os + import platform + + expected = os.environ["EXPECTED_PLATFORM"].rsplit("-", 1)[-1] + actual = platform.machine().lower() + normalized = "arm64" if actual in {"arm64", "aarch64"} else "x64" if actual in {"x86_64", "amd64"} else actual + if normalized != expected: + raise SystemExit(f"runner architecture is {normalized!r}, expected {expected!r}") + PY + - name: Stamp runtime version + shell: bash + env: + RELEASE_VERSION: ${{ inputs.version }} + run: | + version="$RELEASE_VERSION" + if [ -z "$version" ]; then version="0.0.1-dev.${GITHUB_SHA:0:12}"; fi + python packaging/stamp_runtime_version.py --version "$version" + - name: Prepare static OpenSSL for Intel macOS + if: matrix.platform == 'macos-x64' + shell: bash + run: | + brew install openssl@3 + echo "OPENSSL_DIR=$(brew --prefix openssl@3)" >> "$GITHUB_ENV" + echo "OPENSSL_STATIC=1" >> "$GITHUB_ENV" + echo "PIP_NO_BINARY=cryptography" >> "$GITHUB_ENV" + echo "PIP_NO_CACHE_DIR=true" >> "$GITHUB_ENV" + - name: Install locked runtime and packaging toolchain + shell: bash + run: | + python -m pip install \ + -r packaging/requirements-runtime.lock \ + -r packaging/requirements-build.lock + python -m pip install --no-deps . + - name: Run managed runtime tests + shell: bash + env: + PYTHONPATH: ${{ github.workspace }}/src + run: | + python -m pip install -r packaging/requirements-test.lock + python -m pytest -q tests/code4me2_agent tests/test_participant_release.py + - name: Build one-folder bundle + run: python -m PyInstaller --clean --noconfirm packaging/code4me2-agent.spec + # Signing is opt-in. The plugin downloads, verifies (SHA-256 pinned by the + # study server) and launches these archives itself, so neither Gatekeeper + # nor SmartScreen assesses them; signing mainly helps on Windows hosts with + # Smart App Control or strict antivirus. Signing changes the archive bytes, + # so a signed build is a separate runtime version with its own SHA-256. + - name: Sign macOS runtime + if: github.event_name == 'workflow_dispatch' && inputs.sign && startsWith(matrix.platform, 'macos') + shell: bash + env: + CERTIFICATE_P12: ${{ secrets.CODE4ME_MACOS_CERTIFICATE_P12_BASE64 }} + CERTIFICATE_PASSWORD: ${{ secrets.CODE4ME_MACOS_CERTIFICATE_PASSWORD }} + SIGNING_IDENTITY: ${{ secrets.CODE4ME_MACOS_SIGNING_IDENTITY }} + run: | + if [ -z "$CERTIFICATE_P12" ]; then + echo "::error::Missing CODE4ME_MACOS_CERTIFICATE_P12_BASE64 secret" >&2 + exit 1 + fi + if [ -z "$CERTIFICATE_PASSWORD" ]; then + echo "::error::Missing CODE4ME_MACOS_CERTIFICATE_PASSWORD secret" >&2 + exit 1 + fi + if [ -z "$SIGNING_IDENTITY" ]; then + echo "::error::Missing CODE4ME_MACOS_SIGNING_IDENTITY secret" >&2 + exit 1 + fi + keychain="$RUNNER_TEMP/code4me-signing.keychain-db" + certificate="$RUNNER_TEMP/code4me-signing.p12" + if ! printf '%s' "$CERTIFICATE_P12" | base64 -D > "$certificate" || [ ! -s "$certificate" ]; then + echo "::error::CODE4ME_MACOS_CERTIFICATE_P12_BASE64 must decode to a non-empty .p12 file" >&2 + exit 1 + fi + security create-keychain -p temporary "$keychain" + security set-keychain-settings -lut 21600 "$keychain" + security unlock-keychain -p temporary "$keychain" + if ! security import "$certificate" -P "$CERTIFICATE_PASSWORD" -A -t cert -f pkcs12 -k "$keychain"; then + echo "::error::macOS certificate import failed; check the .p12 content and CODE4ME_MACOS_CERTIFICATE_PASSWORD" >&2 + exit 1 + fi + security set-key-partition-list -S apple-tool:,apple: -s -k temporary "$keychain" + security list-keychains -d user -s "$keychain" login.keychain-db + signed=0 + while IFS= read -r binary; do + if file "$binary" | grep -q 'Mach-O'; then + codesign --force --options runtime --timestamp --sign "$SIGNING_IDENTITY" "$binary" + codesign --verify --strict --verbose=2 "$binary" + signed=$((signed + 1)) + fi + done < <(find dist/code4me2-agent -depth -type f) + test "$signed" -gt 0 + # Publicly trusted code-signing keys must live in hardware (HSM, token or a + # cloud signing service) since June 2023, so a PFX secret only works with a + # legacy exportable certificate. + - name: Sign Windows runtime + if: github.event_name == 'workflow_dispatch' && inputs.sign && startsWith(matrix.platform, 'windows') + shell: pwsh + env: + CERTIFICATE_PFX: ${{ secrets.CODE4ME_WINDOWS_CERTIFICATE_PFX_BASE64 }} + CERTIFICATE_PASSWORD: ${{ secrets.CODE4ME_WINDOWS_CERTIFICATE_PASSWORD }} + run: | + if (-not $env:CERTIFICATE_PFX -or -not $env:CERTIFICATE_PASSWORD) { throw "Windows signing secrets are required" } + $certificatePath = Join-Path $env:RUNNER_TEMP "code4me-signing.pfx" + [IO.File]::WriteAllBytes($certificatePath, [Convert]::FromBase64String($env:CERTIFICATE_PFX)) + $password = ConvertTo-SecureString $env:CERTIFICATE_PASSWORD -AsPlainText -Force + $certificate = Import-PfxCertificate -FilePath $certificatePath -CertStoreLocation Cert:\CurrentUser\My -Password $password + $signTool = Get-ChildItem "${env:ProgramFiles(x86)}\Windows Kits\10\bin" -Recurse -Filter signtool.exe | + Where-Object FullName -Match '\\x64\\signtool.exe$' | Sort-Object FullName -Descending | Select-Object -First 1 + if (-not $signTool) { throw "signtool.exe was not found" } + Get-ChildItem dist\code4me2-agent -Recurse -File | Where-Object Extension -In '.exe', '.dll' | ForEach-Object { + & $signTool.FullName sign /fd SHA256 /td SHA256 /tr http://timestamp.digicert.com /sha1 $certificate.Thumbprint $_.FullName + if ($LASTEXITCODE -ne 0) { throw "Signing failed for $($_.FullName)" } + } + - name: Test archived executable and emit native manifest + shell: bash + env: + PYTHONPATH: ${{ github.workspace }}/src + RELEASE_VERSION: ${{ inputs.version }} + MIN_PLUGIN_VERSION: ${{ inputs.minPluginVersion }} + PUBLISH_RELEASE: ${{ inputs.publishRelease }} + run: | + version="$RELEASE_VERSION" + if [ -z "$version" ]; then version="0.0.1-dev.${GITHUB_SHA:0:12}"; fi + metadata=() + if [ "$PUBLISH_RELEASE" = "true" ]; then + metadata+=(--release-base-url "https://github.com/$GITHUB_REPOSITORY/releases/download/runtime-v$version") + fi + if [ -n "$MIN_PLUGIN_VERSION" ]; then metadata+=(--min-plugin-version "$MIN_PLUGIN_VERSION"); fi + python -m research.study.agents.participant_release native \ + --skip-build --version "$version" --platform "${{ matrix.platform }}" \ + --server-commit "$GITHUB_SHA" --output native-release "${metadata[@]}" + - name: Notarize macOS runtime archive + if: github.event_name == 'workflow_dispatch' && inputs.sign && startsWith(matrix.platform, 'macos') + shell: bash + env: + APPLE_ID: ${{ secrets.CODE4ME_APPLE_ID }} + APPLE_TEAM_ID: ${{ secrets.CODE4ME_APPLE_TEAM_ID }} + APPLE_APP_PASSWORD: ${{ secrets.CODE4ME_APPLE_APP_PASSWORD }} + run: | + if [ -z "$APPLE_ID" ]; then echo "::error::Missing CODE4ME_APPLE_ID secret" >&2; exit 1; fi + if [ -z "$APPLE_TEAM_ID" ]; then echo "::error::Missing CODE4ME_APPLE_TEAM_ID secret" >&2; exit 1; fi + if [ -z "$APPLE_APP_PASSWORD" ]; then echo "::error::Missing CODE4ME_APPLE_APP_PASSWORD secret" >&2; exit 1; fi + xcrun notarytool submit "native-release/code4me-agent-${{ matrix.platform }}.zip" \ + --apple-id "$APPLE_ID" \ + --team-id "$APPLE_TEAM_ID" \ + --password "$APPLE_APP_PASSWORD" \ + --wait + - uses: actions/upload-artifact@v4 + with: + name: code4me-agent-${{ matrix.platform }} + path: native-release/* + if-no-files-found: error + retention-days: 1 + + assemble: + needs: build + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: code4me-agent-* + path: release-assets + merge-multiple: true + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: packaging/requirements-runtime.lock + - run: python -m pip install -r packaging/requirements-runtime.lock + - name: Create pinned release metadata + run: | + PYTHONPATH=src python -m research.study.agents.participant_release merge \ + --directory release-assets \ + --output release-assets/code4me-managed-runtime-release.json + - uses: actions/upload-artifact@v4 + with: + name: code4me-managed-runtime-release + path: release-assets/* + if-no-files-found: error + retention-days: 3 + + publish: + if: github.event_name == 'workflow_dispatch' && inputs.publishRelease + needs: assemble + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + name: code4me-managed-runtime-release + path: release-assets + - name: Publish immutable runtime release + env: + GH_TOKEN: ${{ github.token }} + # This job has no checkout, so gh cannot infer the repository from git. + GH_REPO: ${{ github.repository }} + RELEASE_VERSION: ${{ inputs.version }} + run: | + tag="runtime-v$RELEASE_VERSION" + # Assets of a published runtime release are immutable: a re-run must + # fail loudly rather than silently replace a ZIP a plugin already pinned. + if gh release view "$tag" >/dev/null 2>&1; then + echo "release $tag already exists; refusing to replace its assets" >&2 + exit 1 + fi + # Exact declared set: the published ZIPs must be exactly the manifest's + # declared archives (no extra, no missing). + python3 - <<'PY' + import json, pathlib + root = pathlib.Path("release-assets") + manifest = json.loads((root / "code4me-managed-runtime-release.json").read_text()) + declared = sorted(a["archive"] for a in manifest["artifacts"]) + present = sorted(p.name for p in root.iterdir() if p.suffix == ".zip") + if declared != present: + raise SystemExit(f"declared archives {declared} != published zips {present}") + PY + gh release create "$tag" \ + --target "$GITHUB_SHA" \ + --title "Code4Me managed runtime $RELEASE_VERSION" \ + release-assets/*.zip release-assets/code4me-managed-runtime-release.json diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..b9d07c9 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,205 @@ +name: E2E harness + +# The harness lives in code4me2-server/e2e but drives both repositories, so this +# workflow checks the plugin out as a sibling and points the harness at the +# workspace with CODE4ME_E2E_WORKSPACE. +# +# Layers: +# backend - 17 HTTP workflow steps against the disposable Compose stack +# browser - 37 headless Chromium website checks + persisted read models +# plugin - Kotlin IntelliJ fixture + HTTP workflow +# agents - real Goose/Codex ACP probes (only inside `all`, which installs a +# pinned Goose; the harness itself never installs agents) +# all - every layer (needs a graphical runner) +# +# `test` provisions missing prerequisites itself (e.g. e2e/.venv for the native +# agent build); the explicit image/Playwright steps below just keep CI fast and +# its Playwright interpreter explicit. +# +# The full IDE layer runs under Xvfb on a Linux runner. Pin the plugin ref for +# that manual gate so its result identifies the exact pair of source commits. + +on: + workflow_dispatch: + inputs: + layer: + description: Harness layer to run + type: choice + options: + - backend + - browser + - plugin + - all + default: backend + pluginRef: + description: Exact plugin commit SHA to test (required for plugin/all) + required: false + # The harness unit tests run whenever the harness or the backend it drives + # changes; the `layer` job limits the Docker stack to the integration + # branches, pull requests and manual runs. + push: + paths: + - e2e/** + - .github/workflows/e2e.yml + - src/** + - "!src/website/**" + - Dockerfile.cpu + - requirements.txt + - packaging/** + - pyproject.toml + - alembic.ini + pull_request: + paths: + - e2e/** + - .github/workflows/e2e.yml + - src/** + - "!src/website/**" + - Dockerfile.cpu + - requirements.txt + - packaging/** + - pyproject.toml + - alembic.ini + +concurrency: + group: e2e-${{ github.ref }} + cancel-in-progress: false + +jobs: + harness-tests: + name: Harness unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: code4me2-server + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Harness regression tests (no Docker or IDE) + working-directory: code4me2-server/e2e + run: python -m unittest discover -s tests -v + + layer: + name: ${{ inputs.layer || 'backend' }} layer + # Feature-branch pushes stop at the unit tests; dispatch the workflow (or + # open a pull request) to run the stack there. + if: github.event_name != 'push' || github.ref_name == 'test_draft' || github.ref_name == 'main' + runs-on: ubuntu-latest + needs: harness-tests + steps: + - uses: actions/checkout@v4 + with: + path: code4me2-server + - name: Require an explicit plugin revision for IDE and fixture tests + if: ${{ inputs.layer == 'all' || inputs.layer == 'plugin' }} + env: + PLUGIN_REF: ${{ inputs.pluginRef }} + run: '[[ "$PLUGIN_REF" =~ ^[0-9a-fA-F]{40}$ ]]' + - name: Check out the plugin as a workspace sibling + uses: actions/checkout@v4 + with: + repository: AISE-TUDelft/code4me2 + ref: ${{ inputs.pluginRef }} + path: code4me2 + # Public repositories work with the default token; private ones need a + # PLUGIN_CHECKOUT_TOKEN repository secret. + token: ${{ secrets.PLUGIN_CHECKOUT_TOKEN != '' && secrets.PLUGIN_CHECKOUT_TOKEN || github.token }} + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - uses: actions/setup-java@v4 + if: ${{ inputs.layer == 'all' || inputs.layer == 'plugin' }} + with: + distribution: temurin + java-version: "25" + - uses: actions/setup-node@v4 + if: ${{ inputs.layer == 'browser' }} + with: + node-version: "22" + - name: Build the disposable backend image + run: docker build -f code4me2-server/Dockerfile.cpu -t code4me2-server-backend:latest code4me2-server + - name: Install native and graphical test dependencies + if: ${{ inputs.layer == 'all' || inputs.layer == 'plugin' }} + run: | + sudo apt-get update + sudo apt-get install -y xvfb + python -m pip install -r code4me2-server/packaging/requirements-runtime.lock \ + -r code4me2-server/packaging/requirements-build.lock + python -m pip install --no-deps -e ./code4me2-server -e ./code4me2/telemetry-acp-proxy + # Participants bring their own Goose, so the harness only discovers one; on + # this throwaway runner the agents layer gets the release the harness + # targets (Goose 1.51), checked against the digest GitHub publishes for it. + - name: Install the pinned Goose for the agents layer + if: ${{ inputs.layer == 'all' }} + env: + GOOSE_URL: https://github.com/aaif-goose/goose/releases/download/v1.51.0/goose-x86_64-unknown-linux-gnu.tar.bz2 + GOOSE_SHA256: de5bf71c6bf470fbb55276de871bbbd65c844aa066d3b09b1552da83f8e367e2 + run: | + curl -fsSL --retry 3 -o "$RUNNER_TEMP/goose.tar.bz2" "$GOOSE_URL" + echo "$GOOSE_SHA256 $RUNNER_TEMP/goose.tar.bz2" | sha256sum -c - + mkdir -p "$RUNNER_TEMP/goose" + tar -xjf "$RUNNER_TEMP/goose.tar.bz2" -C "$RUNNER_TEMP/goose" + "$RUNNER_TEMP/goose/goose" --version + echo "CODE4ME_E2E_GOOSE_EXECUTABLE=$RUNNER_TEMP/goose/goose" >> "$GITHUB_ENV" + - name: Install Playwright for the browser layer + if: ${{ inputs.layer == 'browser' }} + run: | + python -m venv "$RUNNER_TEMP/browser-venv" + "$RUNNER_TEMP/browser-venv/bin/python" -m pip install --quiet playwright + "$RUNNER_TEMP/browser-venv/bin/python" -m playwright install --with-deps chromium + - name: Run the harness layer + working-directory: code4me2-server/e2e + env: + CODE4ME_E2E_WORKSPACE: ${{ github.workspace }} + CODE4ME_E2E_BROWSER_PYTHON: ${{ inputs.layer == 'browser' && format('{0}/browser-venv/bin/python', runner.temp) || '' }} + E2E_LAYER: ${{ inputs.layer || 'backend' }} + # The runner's Linux Docker engine maps host.docker.internal to the + # bridge gateway, so the backend reaches the stub provider only when it + # listens on all interfaces. + run: | + if [ "$E2E_LAYER" = all ] || [ "$E2E_LAYER" = plugin ]; then + CODE4ME_E2E_PYTHON="$(command -v python)" + export CODE4ME_E2E_PYTHON + xvfb-run -a python -m code4me_e2e test --layer "$E2E_LAYER" --set stack.stub_host=0.0.0.0 --json + else + python -m code4me_e2e test --layer "$E2E_LAYER" --set stack.stub_host=0.0.0.0 --json + fi + - name: Record successful full E2E revision pair + if: ${{ inputs.layer == 'all' }} + run: | + python - <<'PY' + import json + import os + import subprocess + from pathlib import Path + + reports = list(Path("code4me2-server/e2e/runs").glob("*/report.json")) + if len(reports) != 1: + raise SystemExit(f"expected one E2E report, found {len(reports)}") + report = json.loads(reports[0].read_text()) + if not report.get("steps") or report.get("failed_step") or report.get("blocked_by") or any( + step.get("status") != "PASS" for step in report["steps"] + ): + raise SystemExit("full E2E report is incomplete or failed") + plugin_commit = subprocess.check_output( + ["git", "-C", "code4me2", "rev-parse", "HEAD"], text=True + ).strip() + Path("e2e-evidence.json").write_text(json.dumps({ + "layer": "all", + "server_commit": os.environ["GITHUB_SHA"], + "plugin_commit": plugin_commit, + }) + "\n") + PY + - uses: actions/upload-artifact@v4 + if: ${{ inputs.layer == 'all' }} + with: + name: code4me-e2e-evidence + path: e2e-evidence.json + if-no-files-found: error + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@v4 + with: + name: code4me-e2e-report + path: code4me2-server/e2e/runs/*/report.json + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index c0569b5..9f01194 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,31 @@ .DS_Store -node_modules -.venv -.idea -secrets +.env +.env.* +!.env.example +node_modules/ +.venv/ +*.egg-info/ +.idea/ +secrets/ **/__pycache__/ -.hypothesis -.cache -data -.coverage \ No newline at end of file +.hypothesis/ +.cache/ +data/ +backups/ +.coverage + +# Native/PyInstaller and package build output +build/ +dist/ +*.py[cod] + +# Test, lint, type-check and coverage caches +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +htmlcov/ +*.md +# Design and operations documents are part of the repository (review E-05): +# a `git clean` must never be able to delete them. +!README.md +!docs/**/*.md diff --git a/Dockerfile.cpu b/Dockerfile.cpu new file mode 100644 index 0000000..3e229b5 --- /dev/null +++ b/Dockerfile.cpu @@ -0,0 +1,34 @@ +# CPU/no-GPU build for local development on machines without an NVIDIA GPU +# (e.g. Apple Silicon / aarch64). The default `Dockerfile` targets a CUDA host +# and its `nvidia-container-toolkit`/`libnvidia-ml-dev` apt packages have no +# arm64 build, so it cannot be built here at all. This variant drops every +# NVIDIA-specific step; `transformers`/`torch` fall back to CPU automatically +# when no GPU is visible, which is all the agent subsystem needs (it talks to +# an external OpenAI-compatible upstream, not a locally loaded model). +# +# Used by docker-compose.dev-arm.yaml, not by the production docker-compose.yml. +FROM continuumio/miniconda3:latest + +WORKDIR /app + +# argon2-cffi (and a few other deps) build C extensions from source on arm64 +# since no prebuilt wheel exists for this platform/Python combo. +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + libffi-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN conda create -y -n myenv python=3.10 && conda clean -a -y +RUN /bin/bash -c "source activate myenv && conda install -y -c conda-forge huggingface_hub[cli] && conda clean -a -y" + +COPY requirements.txt . +# On arm64 the default PyPI torch wheel is the CUDA build, which pulls ~4 GB of +# nvidia-*/triton wheels that are useless without an NVIDIA GPU. Install the CPU +# build first (keep the pin in sync with requirements.txt) so the requirements +# install finds torch already satisfied. +RUN /bin/bash -c "source activate myenv && pip install --no-cache-dir --index-url https://download.pytorch.org/whl/cpu 'torch>=2.7,<3'" +RUN /bin/bash -c "source activate myenv && pip install --no-cache-dir -r requirements.txt" +RUN /bin/bash -c "source activate myenv && pip install uvicorn" + +COPY . . diff --git a/README.md b/README.md index 184d6c3..e3042b0 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![Docker](https://img.shields.io/badge/docker-ready-blue.svg)](https://docker.com) [![License](https://img.shields.io/badge/License-Apache_2.0-brightgreen.svg)](https://www.apache.org/licenses/LICENSE-2.0) -An advanced AI-powered code completion platform, featuring ghost text suggestions, multi-model inference, a context-aware chat assistant, real-time WebSocket communication, and collaborative development tools suitable for conducting empirical studies on developer behaviour. +An advanced AI-powered code completion platform, featuring ghost text suggestions, multi-model inference, a context-aware chat assistant, an autonomous coding agent (ACP), real-time WebSocket communication, and collaborative development tools suitable for conducting empirical studies on developer behaviour. ## 🎯 Project Overview @@ -14,6 +14,7 @@ Code4me V2 is a research platform that combines transformer models with real-wor - **Real-time Code Completion**: Multi-model AI inference with WebSocket streaming - **Collaborative Development**: Multi-user project management and session tracking +- **Autonomous Coding Agent**: A locally-launched ACP (Agent Client Protocol) agent, with server-assigned model/provider/tools for A/B study arms — see "Agent Subsystem (ACP)" under System Architecture below - **Advanced Analytics**: Comprehensive telemetry and behavioral analysis in a dedicated analysis platform - **Research Platform**: Ground truth collection and model evaluation tools @@ -69,8 +70,16 @@ EMAIL_PORT=587 EMAIL_USERNAME=your_email@gmail.com EMAIL_PASSWORD=your_app_password EMAIL_FROM=noreply@code4me.com + +# Agent Subsystem (ACP) — upstream provider for the coding agent. +# Defaults to a local Ollama instance (no key, no cost). See .env.example +# for the full list of agent variables and the "Agent Subsystem" section below. +AGENT_UPSTREAM_BASE_URL=http://localhost:11434/v1 +AGENT_UPSTREAM_API_KEY= ``` +See [`.env.example`](.env.example) for the complete, documented set of agent-specific variables (per-provider API keys, Codex reasoning effort override, etc.). + ### 3. Deploy with Docker ```bash # Start all services @@ -120,6 +129,64 @@ docker-compose ps - **Project Chat System**: `/api/ws/chat` - Collaborative chat functionality - **Multi-file Context**: `/api/ws/multi-file-context` - Context updates +#### 🤖 Agent Subsystem (ACP) +The `backend` service is also a relay + telemetry sink for an autonomous coding agent (it does not run agent inference loops itself): + +- **Goose** (participant-installed) is pointed at the research inference gateway, `POST /api/research/inference/v1/chat/completions`, with a per-participant inference capability as its API key; the backend forwards the call to the study's OpenAI-compatible provider connection with the server-held key and meters it against the participant's budget. **Codex** signs in with the participant's ChatGPT account and is neither relayed nor metered. (The developer-only path still relays through `POST /api/agent/inference`.) +- **The built-in `code4me2-agent`** runs as a separate local OS process, launched by the IDE plugin, speaking ACP over stdio. It authenticates via a grant → session handoff: the plugin calls `POST /api/acp/grant`, the agent process exchanges it for a bearer token at `POST /api/acp/session/exchange`, then fetches its assigned model/provider/tools from `GET /api/acp/agent-config`. +- An **agent profile** (`agent_profile` table) defines a runtime + provider + model + tools + approval policy — used as an A/B study arm. An **agent assignment** is a sticky, server-authoritative per-user draw; the client never self-selects. +- Provider API keys are never stored in the database — an administrator's *provider connection* stores only the *name* of an environment variable (`secret_ref`), resolved from the backend's own environment at request time. See [`.env.example`](.env.example). +- **Participant budgets.** Goose and built-in arms spend from the study's shared key, so every enrollment carries a USD budget (study default, per-participant top-ups from the website). Each call reserves its worst-case cost before it is forwarded and settles the actual usage afterwards; a participant can never exceed their budget. Prices per model live on the provider connection; see `docs/research-platform/RELEASES.md` for the fail-closed rollout notes. +- Installing and running the local `code4me2-agent` CLI is a separate, standalone step — see "Running the Agent CLI (`code4me2-agent`)" under Development below. + +#### 🧩 Provider-backed classic chat/completion models + +The classic endpoints (`POST /api/chat/request`, `POST /api/completion/request`) can be served by an OpenAI-compatible provider (OpenRouter, Ollama, OpenAI, Groq, vLLM) instead of a local HuggingFace model. A `model_name` row opts in through its `model_parameters` JSON; every other row keeps the local path unchanged. **No plugin change is required** — the plugin keeps sending `model_ids` and the server resolves the row. + +**Default:** the rows the plugin uses by default are seeded provider-backed through OpenRouter (key from `OPENROUTER_API_KEY`): id 1 `deepseek-ai/deepseek-coder-1.3b-base` (completion) is answered by `mistralai/codestral-2508`, and id 3 `mistralai/Ministral-8B-Instruct-2410` (chat) by `mistralai/ministral-8b-2512`. Rows keep their names; `provider_model` records the model that answers. The other rows (StarCoder2, Mellum) run locally and are used only when explicitly chosen. `CLASSIC_MODELS_ENABLED=false` refuses the classic endpoints entirely (HTTP 503) without loading anything. + +```json +{ + "provider": "openai_compatible", + "kind": "chat", // "chat" | "completion"; default: "chat" for *instruct* model names, else "completion" + "base_url": "https://openrouter.ai/api/v1", + "api_key_ref": "OPENROUTER_API_KEY", // optional; name of the env var holding the key (never the key itself) + "provider_model": "mistralai/ministral-8b-2512", // optional; default = model_name + "endpoint": "chat", // "chat" (chat-completions) | "completions" (legacy /completions); default "chat" + "max_new_tokens": 256, + "temperature": 0.2, + "top_p": 0.95, + "timeout_seconds": 60 +} +``` + +- `base_url` is required. `api_key_ref` is optional (a local Ollama needs no key), and like agent profiles it stores only the environment-variable *name*: the value is read from the backend's environment at request time and is never stored or logged. A referenced variable that is missing is a hard error — there is no unauthenticated fallback. +- `kind` selects the model class (chat vs. FIM completion) and defaults from the model name. `endpoint` selects the wire format for completion rows: `chat` sends one system message plus the formatted FIM prompt as a user message, `completions` posts the formatted prompt to `/completions` with `stop` sequences. +- Provider rows report no `confidence` (stored as NULL and left out of the calibration and model analytics) and empty `logprobs` (token logprobs exist only on the local path). A database created before this change needs `ALTER TABLE had_generation ALTER COLUMN confidence DROP NOT NULL;` once (the schema has a single consolidated revision, so existing databases are not migrated automatically); until then provider generations fail to save. +- Unknown `model_parameters` keys are rejected, so a typo cannot silently change the wire shape. + +A database seeded before this default keeps local rows; adopt the default once with: + +```sql +UPDATE model_name +SET model_parameters = '{"provider": "openai_compatible", "kind": "completion", "base_url": "https://openrouter.ai/api/v1", "api_key_ref": "OPENROUTER_API_KEY", "provider_model": "mistralai/codestral-2508", "max_new_tokens": 64}' +WHERE model_name = 'deepseek-ai/deepseek-coder-1.3b-base'; +UPDATE model_name +SET model_parameters = '{"provider": "openai_compatible", "kind": "chat", "base_url": "https://openrouter.ai/api/v1", "api_key_ref": "OPENROUTER_API_KEY", "provider_model": "mistralai/ministral-8b-2512", "max_new_tokens": 256}' +WHERE model_name = 'mistralai/Ministral-8B-Instruct-2410'; +``` + +To host a row locally instead (a HuggingFace model on this server; practical only with a GPU), set it back explicitly: + +```sql +UPDATE model_name SET model_parameters = '{"max_new_tokens": 64}' +WHERE model_name = 'deepseek-ai/deepseek-coder-1.3b-base'; +UPDATE model_name SET model_parameters = '{"max_new_tokens": 256}' +WHERE model_name = 'mistralai/Ministral-8B-Instruct-2410'; +``` + +Set the referenced variable in `.env` (e.g. `OPENROUTER_API_KEY=...`) and restart the `backend` service. The `backend` container must be able to reach `base_url`; a host-run Ollama is `http://host.docker.internal:11434/v1` from inside Docker. The live end-to-end check is `E2E_CLASSIC_PROVIDER=1 python3 -m unittest tests.test_classic_provider_e2e -v`, run from `e2e/`. + #### 📊 Analytics & Telemetry - **Behavioral Analytics**: Typing patterns, acceptance rates, interaction timings - **Performance Metrics**: Model response times, accuracy measurements @@ -131,7 +198,7 @@ docker-compose ps ### Authentication & User Management ```http POST /api/user/create/ # Create new user account -POST /api/user/authenticate/ # User login (email/password or OAuth) +POST /api/user/authenticate/ # User login (email/password) GET /api/user/get # Get current user profile PUT /api/user/update/ # Update user profile DELETE /api/user/delete/ # Delete user account @@ -174,6 +241,36 @@ GET /api/chat/get/{page_number} # Paginated chat history DELETE /api/chat/delete/{chat_id} # Delete chat session ``` +### Agent Subsystem (ACP) +```http +# Profiles & assignments (admin auth) +GET /api/agent/profiles # List agent profiles +POST /api/agent/profiles # Create agent profile +GET /api/agent/available-tools # List tools an adapter can expose +GET /api/agent/registry # List available adapter frameworks +GET /api/agent/assignments # List per-user profile assignments + +# Self-report ingestion & memory (ACP bearer auth, from the local agent process) +POST /api/agent/events/ingest # Ingest agent run/edit telemetry +GET /api/agent/runs/{run_id} # Fetch a stored agent run +GET /api/agent/memory/{session_id} # Read persisted agent memory +PUT /api/agent/memory/{session_id} # Update persisted agent memory +DELETE /api/agent/memory/{session_id} # Delete persisted agent memory + +# Task lifecycle & inference relay (plugin session-cookie auth) +POST /api/agent/task # Start an agent task +POST /api/agent/task/{task_id}/close # Close an agent task +POST /api/agent/task/{task_id}/telemetry # Report task-level telemetry +POST /api/agent/inference # Relay a chat-completion call to the assigned upstream + +# ACP grant handoff (bootstraps the locally launched agent process) +POST /api/acp/grant # Issue a single-use grant (plugin -> backend) +POST /api/acp/session/exchange # Exchange a grant for a bearer session +POST /api/acp/session/validate-or-refresh # Validate/refresh an ACP session +GET /api/acp/agent-config # Fetch assigned model/provider/tools +POST /api/acp/persistent-auth-token # Issue a long-lived token for unattended runs (admin only) +``` + ### Real-time WebSocket Endpoints ```http WS /api/ws/completion # Real-time completion streaming @@ -201,12 +298,13 @@ curl -s \ ## 🗄️ Database Architecture ### Core Tables -- **Users**: Authentication, profiles, and OAuth integration +- **Users**: Email/password authentication and profiles - **Projects**: Collaborative workspaces with multi-file context - **Sessions**: Development session tracking and analytics - **Completions**: AI-generated suggestions with performance metrics - **Telemetry**: Comprehensive usage analytics for research - **Context**: Multi-file code context with vector embeddings +- **Agent tables**: `agent_profile`, `agent_profile_assignment`, `agent_task`, `agent_event`, `agent_edit`, `agent_memory`, `study_agent_profile` — coding agent A/B profiles, per-user assignments, task lifecycle, and self-reported telemetry/edits ### Key Features - **PostgreSQL + pgvector**: Vector similarity search for code context @@ -214,6 +312,22 @@ curl -s \ - **Migration System**: Hybrid SQL initialization + Alembic migrations - **Comprehensive Indexing**: Optimized for high-volume analytics queries +### Running Migrations +The agent/research tables (and any future schema change) are applied via the hybrid migration manager, not automatically on startup: +```bash +# First-time setup (initializes from init.sql + stamps Alembic tracking) +python src/database/migration/migration_manager.py init + +# Apply pending migrations +python src/database/migration/migration_manager.py migrate + +# Check current revision / history +python src/database/migration/migration_manager.py status +``` +There is a single consolidated Alembic revision, `8a0084080b46_consolidated_schema.py` (fresh-schema only): it creates the research/agent schema and refuses to run against a database that already holds the previous research tables, so create a new database/volume instead of upgrading in place. Default agent profiles (`default-code4me2-agent`, `default-goose`, `default-codex`) are **not** created by a migration; they are seeded by the development seeder (`scripts/dev/seed_local_dev.sh`, see **Research Platform** below). Only `default-code4me2-agent` is seeded active by default; the Goose/Codex profiles are seeded inactive (BYOA) until you provide those external binaries. + +**Current default profile (initial testing config)**: `default-code4me2-agent` points at OpenRouter (`https://openrouter.ai/api/v1`) using the free `cohere/north-mini-code:free` model, with `api_key_ref=OPENROUTER_API_KEY`. Set `OPENROUTER_API_KEY` in `.env` once you have a key — until then, requests are forwarded unauthenticated and OpenRouter will 401. Update the profile via `PUT /api/agent/profiles/{profile_id}` (or the admin UI) to switch models/providers later. + ## ⚡ Asynchronous Processing ### Celery Task Queues @@ -306,6 +420,24 @@ Notes: - The API runs on `SERVER_HOST`/`SERVER_PORT` (default `0.0.0.0:8008`). - When running via Docker, `nginx` serves the website on `http://localhost:8000` and proxies `/api/*` to the backend. +### Running the Agent CLI (`code4me2-agent`) +`code4me2-agent` is a **separate, standalone package** (`pyproject.toml` at the repo root) — it is not installed into the backend's `requirements.txt`/Docker image, since it runs as its own local OS process launched by the JetBrains plugin (or manually, for development), not inside the FastAPI server. + +```bash +# Install the agent CLI package (from the repo root, ideally in its own venv) +pip install -e . + +# Write default config files to ~/.code4me/ and print IDE setup instructions +code4me2-agent --setup + +# Run it directly over stdio (normally the IDE plugin does this for you) +code4me2-agent +``` +Notes: +- The agent has **no hardcoded model/provider/API key** — after it authenticates via the ACP grant handoff (`POST /api/acp/grant` → `POST /api/acp/session/exchange`), it fetches `GET /api/acp/agent-config` and the backend's assigned agent profile overrides its local config. This is deliberate: a stale local config can't silently override a study assignment. +- The seeded default profile currently points at OpenRouter (`cohere/north-mini-code:free`) — set `OPENROUTER_API_KEY` in `.env` before it will authenticate. If you instead assign a local-Ollama profile, make sure `ollama serve` is running before launching the agent. +- Local-process environment variables (set by whatever launches the agent, e.g. the plugin): `CODE4ME_ACP_BACKEND_URL`, `CODE4ME_ACP_GRANT`, `CODE4ME_ACP_TOKEN`, `CODE4ME_ACP_LOG_SECRETS`, `CODE4ME_AGENT_LOG_LEVEL`, `CODE4ME_MODEL_REQUEST_LIMIT`, `CODE4ME_MODEL_REQUEST_WINDOW_SECONDS`, `CODE4ME_BACKEND_429_MAX_RETRIES`, `CODE4ME_BACKEND_429_RETRY_SLEEP`, `CODE4ME_RATE_LIMIT_STATE_PATH`. These are distinct from the backend's own `AGENT_*`/`*_API_KEY` variables in `.env`. + ### Frontend Development ```bash cd src/website @@ -340,9 +472,18 @@ src/ main.py # FastAPI entrypoint (serves /api, docs at /docs) App.py # Application singleton (DB, Redis, Celery, Models) backend/routers/ # REST and WS route modules (mounted at /api) + agent/ # Agent profiles, consent, ingest, memory (/api/agent) + agents.py # Agent task lifecycle + inference relay (/api/agent) + acp/ # ACP grant handoff (/api/acp) celery_app/ # Celery setup and task modules + agents/ # Backend-side agent logic: provider routing, registry, + # normalization, event/telemetry ingestion (imported + # by backend/routers/agent* — not a Celery task) + code4me2_agent/ # Standalone ACP agent CLI (separate package, see + # pyproject.toml — not part of the backend image) database/ # SQLAlchemy models, CRUD, migrations, pgvector website/ # React frontend (served via nginx at :8000) + src/pages/AgentProfiles.js, AgentAssignments.js # Admin-only agent management UI ``` ## 📊 Monitoring & Health Checks @@ -397,6 +538,14 @@ docker-compose logs -f celery-worker **Common 401/403 Causes** - Missing `session_token`/`project_token` cookies when calling protected endpoints. - Using POST for `/api/session/acquire` (it is GET). +- Agent endpoints under `/api/agent/events`, `/api/agent/memory` require an ACP bearer token (from `/api/acp/session/exchange`), not a session cookie. +- `/api/agent/profiles` and `/api/agent/assignments` require an admin user. + +**Agent Subsystem Issues** +- **401 from OpenRouter**: the seeded default profile (`default-code4me2-agent`) uses OpenRouter with `api_key_ref=OPENROUTER_API_KEY`. Set `OPENROUTER_API_KEY` in `.env` and restart `backend` — until it's set, calls are forwarded unauthenticated and OpenRouter rejects them. +- **Agent can't reach Ollama from inside Docker** (only relevant if you switch a profile back to a local Ollama `base_url`): `http://localhost:11434/v1` resolves to the *container*, not your host. Either run the backend outside Docker for agent development, or set that profile's `base_url` (or the `AGENT_UPSTREAM_BASE_URL` fallback) to `http://host.docker.internal:11434/v1` (the `backend` service already maps `host.docker.internal` via `extra_hosts` in `docker-compose.yml`). +- **Agent tables missing**: run `python src/database/migration/migration_manager.py migrate` — the agent tables are not created by `init.sql`, only by Alembic migrations. **Default profiles absent**: run the development seeder (`scripts/dev/seed_local_dev.sh`); the default agent profiles are not created by a migration. +- **Agent CLI won't authenticate**: check that the launching process (normally the JetBrains plugin) set `CODE4ME_ACP_BACKEND_URL`, `CODE4ME_ACP_GRANT`, and `CODE4ME_ACP_TOKEN`, or that the workspace handoff file exists — the grant is single-use and short-lived. ### Debugging Commands ```bash @@ -445,6 +594,15 @@ MODEL_CACHE_DIR=./data/hf MODEL_MAX_NEW_TOKENS=64 MODEL_USE_CACHE=true +# Agent Subsystem (ACP) — see .env.example for the full documented list +AGENT_UPSTREAM_BASE_URL=http://localhost:11434/v1 +AGENT_UPSTREAM_API_KEY= +OLLAMA_API_KEY= +OPENAI_API_KEY= +GROQ_API_KEY= +OPENROUTER_API_KEY= +AGENT_REASONING_EFFORT= + # Security Configuration AUTHENTICATION_TOKEN_EXPIRES_IN_SECONDS=3600 SESSION_TOKEN_EXPIRES_IN_SECONDS=3600 @@ -458,17 +616,133 @@ EMAIL_PASSWORD=your_app_password EMAIL_FROM=noreply@code4me.com ``` +## 🧪 Research Platform + +The research platform (Issue 01+) is mounted under `/api/research` and is +documented in [`docs/research-platform/`](docs/research-platform/). It is +admin/operator-facing: compatibility, protocol publication, registry, bootstrap, +and exports record evidence or define experimental intent and are never exposed +to participants. + +```http +# ACP host capability receipts + compatibility gate (admin) +GET/POST /api/research/compatibility/... + +# Immutable StudyProtocolV1 authoring + publication (admin) +GET/POST /api/research/studies/... + +# Agent registry, artifacts, and capability snapshots (admin) +GET/POST /api/research/agents/... + +# Participant identity, consent, and withdrawal +GET/POST /api/research/participants/... + +# Bootstrap manifest retrieval/validation +GET/POST /api/research/bootstrap/... + +# Research session lifecycle (session capability auth) +GET/POST /api/research/sessions/... + +# Canonical telemetry batch ingestion +GET/POST /api/research/telemetry/... + +# Researcher read models + pilot operations/release gate (RBAC/admin) +GET/POST /api/research/operations/... +``` + +### Seed a fresh database from the built runtime + +A fresh database is made immediately usable by importing the runtime **producer +manifest together with its exact archive bytes** (never a hand-typed digest), +checking the producer tests for its built platforms, pinning +`default-code4me2-agent` to it, marking the participant-installed profiles +(`default-goose`/`default-codex`) as BYOA, and publishing one study revision with +a working `session_policy`. + +```bash +cd code4me2-server +MANIFEST=dist/release-1.2.3/native-macos-arm64.json scripts/dev/seed_local_dev.sh +``` + +`seed_local_dev.sh` is a thin driver: it stages the built archives into the +running backend container and imports the producer manifest unchanged. Generate +it first with the [native release command](docs/research-platform/RELEASES.md). Every declared archive's real `sha256`/size are recomputed and checked, +so no digest or size is ever typed by hand. `MANIFEST` is required and names the +producer manifest with its ZIP next to it; the plugin repository ships no agent. + +The same verified import is the only way a release enters the catalogue, and is +exposed to CI as an admin **multipart** endpoint: + +```bash +curl -X POST http://localhost:8008/api/research/agents/releases/import \ + -H "Cookie: auth_token=$TOKEN" \ + -F 'manifest= + enrollment_status: ACTIVE + study_id: + revision_id: + agent_id: code4me-synthetic-agent + release_id: code4me-synthetic-agent-synthetic-1 + artifact_digest: sha256:<64 hex> + login email: participant@example.com + login password: Password123 +``` + +The script is deterministic and safe to re-run: the same arguments resolve the +same account, release, study, published revision and enrollment instead of +piling up duplicates. A password is only printed when this run created the +account (`--create-account`); an existing account is left untouched. The release +is registered `QUALIFIED` and immutable — changing `--artifact-digest` without a +new `--release-id` is rejected rather than mutating a published release. + ## 📖 Documentation - **API Documentation**: http://localhost:8008/docs (when running) - **Database Schema**: See `src/database/resources/documentation/` - **Authentication Workflows**: See `src/backend/resources/documentation/` +- **Research Platform**: See [`docs/research-platform/`](docs/research-platform/) ## 🔒 Production Notes -- Restrict CORS origins and cookies in production. -- Configure strong `DB_PASSWORD`, rotate secrets, and set per-environment `.env`. -- Place `nginx` behind TLS termination and enable HSTS. -- Consider scaling `celery-worker` replicas per GPU availability; update device IDs accordingly. +- Deploy with `docker-compose.prod.yml` (CPU images, static website behind nginx, + health checks, bounded logs, nightly backups); the walkthrough is + [`docs/research-platform/PRODUCTION_DEPLOYMENT.md`](docs/research-platform/PRODUCTION_DEPLOYMENT.md). +- `BOOTSTRAP_SIGNING_SECRET`, `REDIS_PASSWORD` and `PUBLIC_ORIGIN` are required; + `.env.example` lists the whole inventory. Never reuse a development `.env`. +- Place the `website` container behind TLS termination and enable HSTS at the edge; + cookies are `Secure` (`COOKIE_SECURE=true`) and the plugin refuses plain HTTP. +- `docker-compose.yml` is the GPU stack for the local Hugging Face rows only. ## 🤝 Contributing diff --git a/benchmarks/swebench/.gitignore b/benchmarks/swebench/.gitignore new file mode 100644 index 0000000..eae72af --- /dev/null +++ b/benchmarks/swebench/.gitignore @@ -0,0 +1,5 @@ +runs/ +.venv/ +*.egg-info/ +.venv-rebench/ +.venv-mini/ diff --git a/benchmarks/swebench/README.md b/benchmarks/swebench/README.md new file mode 100644 index 0000000..582580e --- /dev/null +++ b/benchmarks/swebench/README.md @@ -0,0 +1,353 @@ +# SWE-bench runs for the Code4Me agent + +This folder runs the packaged Code4Me agent (`code4me2_agent`) on SWE-bench Verified tasks +and keeps the agent's own telemetry for every task. It does not use the plugin, the IDE or +the ACP transport. A small driver calls the agent core (`EchoAgentCore.handle_prompt`) +directly, inside each task's container. + +It is self-contained: +- It has its own `pyproject.toml` and virtual environment. +- Nothing outside this folder imports it. +- The server's test suite does not collect it. + +## How a task runs + +1. **Runtime.** `code4me-swebench runtime` builds the agent wheel from this checkout. It + installs the wheel, together with the release's locked dependencies + (`packaging/requirements-runtime.lock`), on a portable Python 3.13. The result goes into a + Docker volume named after the content hash of the wheel and the lock file. + - Why not the native release bundle: it is built on Ubuntu 24.04 and needs a newer glibc + than the SWE-bench images (Ubuntu 22.04) provide. +2. **Container.** For each task, the official instance image + (`swebench/sweb.eval.x86_64.`) starts with the runtime mounted read-only at + `/opt/code4me`. +3. **Agent run.** `driver.py` runs the agent once in `/testbed`, the repository at the task's + base commit. + - Commands run with the image's `testbed` conda environment on `PATH`, exactly where the + grading tests run. + - The prompt contains the issue text only: no hints and no tests (see `prompt.py`). + - The `ask_user` tool is removed and approvals are automatic. +4. **Patch.** Everything the agent changed is collected as `git diff --cached --binary` + against the starting commit. That diff becomes the prediction. +5. **Grading.** The official harness (`swebench.harness.run_evaluation`, swebench 5.0.2) + grades the prediction in a fresh container. + +## Network sandbox + +Task containers get no internet. Before 2026-10-09 they ran on Docker's default network, and +both agents used it to fetch the upstream fix: `pip download` of a newer release, GitHub pull +request diffs, `git fetch`. Results from those runs are not valid. + +- **Our agent** runs inside the container and needs its model API. The container therefore joins + the `--internal` network `code4me-swebench-internal`, which has no route out. + - Its only exit is `code4me-swebench-proxy`, a stdlib CONNECT proxy (`egress_proxy.py`) that + allows the model host on port 443 and nothing else. + - The agent and every command it runs get `HTTP(S)_PROXY` pointing at the proxy. + - Refused attempts are logged (`docker logs code4me-swebench-proxy`). Each task records its own + in `host.json` as `egress_denied`. +- **mini-swe-agent** calls the model from the host, so its containers run with `--network none`. + +## Telemetry + +Each task writes `trace.jsonl`. It is the same `code4me.agent.event.v1` event stream that +the agent uploads to `/api/agent/events/ingest` in a study, with raw capture on. It includes: +- every model call: model, finish reason, token usage, provider cache hits, latency and call + purpose, plus the raw request and response; +- every tool call: arguments, result, status, duration and denials; +- commands with parsed test-runner summaries; +- run start and end, with the stop reason. + +`telemetry.json` summarises each trace. `report.json` and `instances.csv` aggregate the run. + +`code4me-swebench check [RUN ...]` validates every trace. It checks that: +- each trace has one run and one schema; +- sequence numbers are 1..N and timestamps are in order; +- there is a start and an end; +- every model request is answered: by `agent.model.completed`, by a main-loop failure + (`agent.adapter.loop_failed`), or by a failed optional side call counted in + `agent.run.completed` (`metrics.failed_side_calls`); +- every tool call has an outcome; +- parent links resolve; +- token metrics equal the provider's raw usage; +- the provider key does not appear, when it is set in the environment. + +Agent builds before 2026-10-09 recorded nothing when a model call failed, so a failure shows up +as one unanswered request. + +Traces contain prompts and tool output from public repositories, and they are large. `runs/` +is git-ignored. + +The provider key never reaches disk or the trace. Only its environment variable *name* is +written. The value goes to `docker exec -e NAME`, and the agent's command environment +allowlist keeps it away from the commands the model runs. + +## Requirements + +- Docker that can run `linux/amd64` images. The images are published for amd64 only. + - x86-64 Linux works natively. + - On Apple Silicon, use Colima with `vmType: vz` and `rosetta: true` (this machine's setup). +- Disk: about 1–2 GB per task image. Pass `--remove-images` to delete each image after it is + graded. +- `uv`. +- A provider key in an environment variable. The default is OpenCode Go + (`https://opencode.ai/zen/go/v1`, key in `OPENCODE_GO_API_KEY`). The agent adds the + `x-opencode-session` header that OpenCode requires. + +## Usage + +```bash +cd code4me2-server/benchmarks/swebench +uv venv --python 3.13 .venv && uv pip install --python .venv/bin/python -e '.[test]' +.venv/bin/python -m pytest -q # unit tests (no Docker, no network) + +export OPENCODE_GO_API_KEY=... # never commit it +.venv/bin/code4me-swebench runtime # once per agent change (~30 s) +.venv/bin/code4me-swebench run --run-name v4flash-mini-s1 --subset verified-mini \ + --model deepseek-v4.1-flash --remove-images +.venv/bin/code4me-swebench report --run-name v4flash-mini-s1 +``` + +`run` behaves as follows: +- It is resumable. Finished tasks are skipped, and a run directory refuses different + settings. +- It grades each task as soon as the agent finishes. Pass `--no-eval` to skip grading, then + grade later with `code4me-swebench evaluate`. +- Useful flags: + - `--limit N` and `--instances ID ...` for smoke tests; + - `--workers N` for parallel tasks (keep it at 1 on an 8 GB Colima VM); + - `--max-iterations` (default 250 model calls per task, mini-swe-agent's stock step limit); + - `--context-tokens` (default 400k: trimming should almost never be needed; see below); + - `--task-timeout` (default 10800 s, a guard against hangs); + - `--memory` and `--cpus` per container; + - `--no-self-review` to switch off the agent's end-of-task self-review. + +### Context window + +Agent builds before 2026-10-09 broke provider prompt caching on long tasks. Once a single turn +outgrew the memory window, the agent elided "just enough" old tool output on every call. That +changed the request prefix on every call, so almost every later call was a full cache miss. +Their chars/4 estimate was also about 20% (up to 2×) below the provider's count. + +The agent now does three things: +- calibrates its estimate from the reported usage; +- elides in one step down to 60% of the budget; +- reports the compaction as integer metrics on the next `agent.model.requested` + (`context_elided_units`, `context_tokens_before`/`_after`, `token_scale_milli`). + +No new telemetry event types are added, because the study counts steps as events (constraint +C1 of the 2026-09-26 tiers run). + +Benchmarks also set a 400k window. The run `rebench-v4flash-s1` (100k window, old build) is +kept as evidence of the old behaviour. + +### Agent behaviour in benchmark runs + +The benchmark config sets the agent's `autonomous` flag. IDE sessions never do, and study +profiles cannot. With it set: +- **Continuation.** A turn that ends announcing more work ("Let me…", "I'll…", a trailing colon, + but not "Let me know"), or whose answer was cut off by the output limit, is continued. + - At most two continuations per turn. + - Each needs a tool run since the previous one. + - The request that follows carries `continuation_nudge: true`. + +The following apply in every mode, including IDE sessions, as of 2026-10-09: +- **Reasoning pass-back.** The model's reasoning goes back as `reasoning_content`, exactly as + returned. + - DeepSeek V4/V4.1 models also get `""` on every other assistant message when tools are sent, + as their API requires. + - A server that rejects the field gets nothing more after one retry. +- **Verification.** Only commands that run code count as verification before stopping. Inspection + (`git status/diff`, `ls`, `cat`, `grep`), file moves and package installs do not. +- **Arguments.** Out-of-range size and paging arguments are clamped, and a plain argv string is + split. Both are reported to the model as `argument_notes`. + +### Self-review is off in our main runs + +With DeepSeek V4 Flash, the agent's end-of-task self-review takes 54% of task time. The +model reasons for about 19k tokens on a 2k-token review prompt, and 4 of 20 reviews stalled +until the server cut them off after 20 minutes. The review led to further edits in only +3 of 22 tasks. + +Main runs therefore pass `--no-self-review`. The setting is recorded in `manifest.json` and in +every task's `agent-config.json`, and results must state it. The paired Verified-20 runs +(`verified20-v4flash-s1` with it on, `verified20-v4flash-noreview-s1` with it off) measure +its effect. + +### SWE-rebench + +SWE-rebench (`nebius/SWE-rebench-leaderboard`) builds tasks from fresh GitHub issues each month, +so models are less likely to have trained on them. The same `run` command handles it: + +```bash +.venv/bin/code4me-swebench --dataset nebius/SWE-rebench-leaderboard --split 2026_03 \ + run --run-name rebench-v4flash-s1 --subset rebench-2026_03-newest-50 --remove-images +``` + +- **Grading** uses SWE-rebench's own fork of the harness (pinned commit, separate + `.venv-rebench`, created on first use). The fork is swebench 4.0.3, which conflicts with + swebench 5. +- **Image layout.** SWE-rebench images keep conda in `/opt/conda`, not `/opt/miniconda3`. + The runner finds either, and fails the task if there is no `testbed` environment. +- **Leak files.** Some older SWE-rebench images ship `/issue.md` and + `/swebench_instance.json`. The runner deletes them before the agent starts and records + this in `host.json`. + +### Baseline: mini-swe-agent (optional, not used in the main runs) + +`baseline-mini` runs mini-swe-agent (bash-only, version 2.4.6, stock `swebench.yaml`) on the +same tasks with the same model and grader. It writes the same run-directory layout, so +`report` works on it unchanged. It runs in its own `.venv-mini`. + +Only the following are overridden: +- the model endpoint (`mini-overlay.yaml`); +- the container platform; +- cost tracking, because litellm has no price for these models. + +OpenCode's session header comes from `mini_baseline/opencode_model.py`. Tasks run in batches, +so `--remove-images` can keep disk use bounded. + +```bash +.venv/bin/code4me-swebench --dataset nebius/SWE-rebench-leaderboard --split 2026_03 \ + baseline-mini --run-name rebench-mini-v4flash-s1 --subset rebench-2026_03-newest-50 --remove-images +``` + +Known differences from our runner: +- Both stop at 250 model calls (mini-swe-agent's stock step limit and our default). mini's stock + $3 cost limit never applies here: litellm has no price for these model ids. +- It submits the diff it writes itself (`git diff -- `), where we stage everything + (`git add -A`). + +### Main runs: the SWE-rebench leaderboard protocol + +We compare against the official SWE-rebench leaderboard (swe-rebench.com), not against a baseline of +our own. It publishes results for one fixed scaffold and many models on fixed task windows. Running +our agent with the same model, on the same window and with the same metrics gives a comparison in +which the scaffold is the main difference. `references.json` holds the published numbers, read from +the site on 2026-10-14. + +| | Leaderboard | Our runs | +| --- | --- | --- | +| Tasks | a half-month-aligned window; HF split `2026_03` is exactly 2026-03-01→05-15 (110 tasks) | the same window files (`subsets/rebench-2026_03-*`), dataset revision pinned | +| Runs | 5 per model | `--repeats 5` | +| Metrics | mean resolved rate over the 5 runs (every task counts), SEM = sd/√5, pass@5, tokens and cost per problem | `aggregate`, same formulas | +| Steps | no cap (runs reached 492) | 500 model calls | +| Context | 128k; past that the task fails | 128k; our agent compacts instead | +| Command timeout | 300 s | 300 s | +| Sampling | model defaults (V4: temperature 1.0, top_p 1.0) | sent explicitly | +| Network | open (pip and GitHub reachable) | model API only (stricter) | +| Scaffold | in-house minimal ReAct agent | our agent: the thing being compared | + +**Which model.** The leaderboard's only DeepSeek Flash entry is DeepSeek-V4 Flash [high]. It scores: +- 38.36 ± 0.97 on the full 2026_03 window (pass@5 57.3, $0.069 and 3.0M tokens per problem); +- 36.09 ± 1.63 on its 46-task March window (2026-03-01→04-01, pass@5 56.5), our planned set. +DeepSeek V4.1 Flash has no leaderboard number. For a same-model comparison, run +`deepseek/deepseek-v4-flash`. + +DeepSeek's own API now answers to that name with V4.1, so on OpenRouter V4 Flash comes only from +third-party hosts. Pin one fp8 host, and record which one. Which host the leaderboard used is not +published. + +**OpenRouter pinning.** OpenRouter serves one model from many hosts, which differ in: +- precision (fp4, fp8 or full); +- output limit; +- price. + +By default it picks a cheap host per request, and it may pick a different one on every call. So: +- `--openrouter-upstream SLUG` sends `provider: {only: [SLUG], allow_fallbacks: false, + require_parameters: true}`. Every request goes to that host, or it fails. A run with an + `openrouter.ai` URL and no pin is refused. +- OpenRouter runs also send `reasoning: {effort: "high"}`, `top_p: 1.0` and `temperature: 1.0`. + Override them with `--extra-body` or `--temperature`. +- All of these fields are recorded in `manifest.json` and in each task's `agent-config.json`. +- `telemetry.json` lists the hosts (`upstreams`) that served each task and the billed `cost_usd`. + `check` fails a task if any call names another host, or names none at all. + +```bash +export OPENROUTER_API_KEY="$(cat ~/.config/code4me/openrouter-key)" # file is chmod 600 +OR=(--base-url https://openrouter.ai/api/v1 --api-key-env OPENROUTER_API_KEY + --model deepseek/deepseek-v4-flash --openrouter-upstream deepinfra) +RB=(--dataset nebius/SWE-rebench-leaderboard --split 2026_03) + +# Pre-flight, a few cents: the key, the pin and DeepSeek's reasoning pass-back through OpenRouter. +.venv/bin/code4me-swebench "${RB[@]}" run --run-name lb-preflight \ + --instances tox-dev__tox-3931 --no-self-review --remove-images "${OR[@]}" +.venv/bin/code4me-swebench check lb-preflight # upstreams/cost_usd: telemetry.json + +# Main: 5 independent runs of the 46-task March window, task by task (each image is pulled once). +# For the full 110-task window use --subset rebench-2026_03-0301-0515. +.venv/bin/code4me-swebench "${RB[@]}" run --run-name lb-v4flash \ + --subset rebench-2026_03-0301-0401 --repeats 5 --no-self-review --remove-images "${OR[@]}" +.venv/bin/code4me-swebench check lb-v4flash-r1 lb-v4flash-r2 lb-v4flash-r3 lb-v4flash-r4 lb-v4flash-r5 +.venv/bin/code4me-swebench aggregate --run-name lb-v4flash --reference deepseek-v4-flash-high +``` + +`aggregate` reports the run's window and every 2026_03 window it fully covers, next to the +reference, with the difference and its z score. A difference counts at |z| ≥ 2: +- 46-task March window: roughly 5 points; +- 64-task window: roughly 6 points; +- full 110-task window: roughly 3 points. + +**Cost.** Per task, using our measured token usage (3.9M input tokens, 95% from the cache; 44k +output tokens): + +| Host | Per task | 5 × 46 | 5 × 64 | 5 × 110 | +| --- | --- | --- | --- | --- | +| Baidu fp8 | $0.040 | $9 | $13 | $22 | +| StreamLake fp8 | $0.072 | $17 | $23 | $40 | +| DeepInfra fp8 | $0.093 | $21 | $30 | $51 | +| V4.1 Flash, DeepSeek's own API, off-peak | $0.068 | $16 | $22 | $37 | + +- The leaderboard's own token use (3.0M per problem) would be about 25% lower. +- Hosts change prices; `report.json` shows what was actually billed. +- **Time:** about 10 minutes per task-run with one worker: 5 × 46 takes about 1.6 days, 5 × 64 + about 2.2 days and 5 × 110 about 3.8 days. + +**Caveats.** +- **Unpaired comparison.** The leaderboard publishes aggregates only, no per-task results, for + this window. The comparison is therefore not paired. +- **Contamination.** V4 Flash (released 2026-04-24) carries the leaderboard's contamination flag + on every window except 2026-05-01→05-15 (21 tasks). +- **Interface text.** 24 tasks carry an "Interface" section in the issue text. 15 others have one + in a separate dataset field that our prompt leaves out; whether the leaderboard showed it is + unknown. +- **Network.** Their sandbox was online and ours is offline. That can cost us tasks that need a + package install. + +**Key cap.** Give the key a credit limit in the OpenRouter dashboard. +- An exhausted key answers 402 or 403. Like 401, 404 (no model or pinned host), rate limits and + outages, that counts as a provider outage, not a failed task. +- The task is recorded as `infra_error`, three in a row stop the run, and `requeue` releases them. + +## Subsets (`subsets/`) + +| File | What it is | +| --- | --- | +| `verified-mini.txt` | The public SWE-bench Verified Mini: 50 tasks, django and sphinx only, chosen to track Verified's resolve rate. Comparable with other published Mini numbers, but covers only 2 repositories and skews harder (8 of 50 tasks take an hour or more, against 45 of 500 in the full set). | +| `verified-stratified-20.txt` | 20 tasks with the same stratification (seed 0): the quick check of the setup. | +| `rebench-2026_03-0301-0515.txt` | The SWE-rebench leaderboard window 2026-03-01→05-15: all 110 tasks of split 2026_03 (dataset revision `34d5a588`). The main task set. | +| `rebench-2026_03-0301-0401.txt` | The leaderboard window 2026-03-01→04-01: 46 tasks (V4 Flash: 36.09 ± 1.63). | +| `rebench-2026_03-0401-0515.txt`, `-0415-0515`, `-0501-0515` | Its leaderboard sub-windows: 64, 39 and 21 tasks. Only the last postdates DeepSeek V4's release (2026-04-24). | +| `rebench-2026_03-newest-50.txt` | Earlier runs only: the 50 newest tasks of split 2026_03 (2026-04-10 to 05-12). It is not a leaderboard window, so no official number exists for it. | +| `verified-stratified-50.txt` | 50 tasks drawn from all of Verified, proportional to repository × difficulty (largest remainder, seed 0, `code4me-swebench select`). It covers 10 repositories but needs more image storage. | + +## Reporting + +- The resolve rate counts every task in the subset. Tasks that were not run or not graded + count as unresolved. +- `report.json` gives a Wilson 95% interval. At 50 tasks the interval is roughly ±13 points. +- For claims, use the leaderboard protocol above: 5 runs, `aggregate`, and a published reference + for the same model and window. +- SWE-bench Verified is widely regarded as saturated and partly contaminated. Use it to check + that the harness works, and add a newer benchmark (SWE-rebench, SWE-bench Pro) for + capability claims. + +## Not covered here + +- **Leaderboard windows after 2026-05-15.** The newest window (2026-05-15→07-01, 111 tasks) is + published only as a Harbor dataset, and most of its tasks are not Python. It is the only window + with per-run trajectories, which would allow a paired comparison. V4 Flash was not run on it. +- **The ACP and plugin path.** Driving the same agent over ACP (for example with Harbor's + ACP agent support or the research telemetry proxy) would exercise the transport the study + uses. It is not implemented. +- **Uploading traces to a Code4Me server.** Ingest is idempotent by event id, so traces could + be replayed after a run under a benchmark account. It is not implemented. diff --git a/benchmarks/swebench/code4me_swebench/__init__.py b/benchmarks/swebench/code4me_swebench/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/swebench/code4me_swebench/baseline_mini.py b/benchmarks/swebench/code4me_swebench/baseline_mini.py new file mode 100644 index 0000000..fee3ee5 --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/baseline_mini.py @@ -0,0 +1,200 @@ +"""Baseline: mini-swe-agent (bash-only) on the same tasks, model and grader. + +mini-swe-agent runs with its stock ``swebench.yaml`` config; only the model +endpoint, the container platform and cost tracking are overridden. Tasks run in +batches so images can be deleted after grading (mini-swe-agent keeps them). +Results land in the same run-directory layout as the Code4Me runner, so +``report`` works unchanged; telemetry is mini-swe-agent's trajectory, summarised. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import subprocess +from pathlib import Path +from typing import TYPE_CHECKING +from urllib.parse import urlparse + +import yaml + +from code4me_swebench import docker_cli, evaluate, runner + +if TYPE_CHECKING: + from code4me_swebench.settings import RunSettings + +logger = logging.getLogger(__name__) +MINI_VERSION = "2.4.6" +ROOT = Path(__file__).resolve().parents[1] +MINI_VENV = ROOT / ".venv-mini" +MODEL_CLASS_DIR = ROOT / "mini_baseline" +# mini-swe-agent's exit_status is the exception name when the model call failed. +# These are the model service failing (rate limit, outage), not the agent. +PROVIDER_FAILURES = frozenset({ + "RateLimitError", "APIConnectionError", "ServiceUnavailableError", + "InternalServerError", "Timeout", "APIError", "BadGatewayError", + # The service refusing to serve: bad/exhausted key, no model or upstream. + "AuthenticationError", "PermissionDeniedError", "NotFoundError", +}) + + +def mini_executable() -> Path: + executable = MINI_VENV / "bin" / "mini-extra" + if not executable.exists(): + subprocess.run(["uv", "venv", "--quiet", "--python", "3.12", str(MINI_VENV)], check=True) + subprocess.run(["uv", "pip", "install", "--quiet", "--python", str(MINI_VENV / "bin" / "python"), + f"mini-swe-agent=={MINI_VERSION}"], check=True) + return executable + + +def overlay_config(settings: RunSettings) -> dict: + return { + "model": { + "model_name": f"openai/{settings.model}", + # litellm has no price for these models; the cost limit cannot apply. + "cost_tracking": "ignore_errors", + "model_kwargs": { + "api_base": settings.base_url, + "extra_headers": {"User-Agent": f"code4me-research-baseline/mini-swe-agent-{MINI_VERSION}"}, + **({"temperature": settings.temperature} if settings.temperature is not None else {}), + # The same extra request fields as our agent (e.g. OpenRouter's upstream pin). + **({"extra_body": dict(settings.extra_body)} if settings.extra_body else {}), + }, + }, + "environment": { + # The model is called from the host, so the task container needs no + # network at all: no upstream fixes from GitHub or PyPI. + "run_args": ["--rm", "--platform", settings.platform, "--network", "none"], + "pull_timeout": 1_200, + }, + } + + +def model_class_args(settings: RunSettings) -> list[str]: + """OpenCode needs a per-conversation session header; other endpoints get stock litellm.""" + host = (urlparse(settings.base_url).hostname or "").lower() + if host == "opencode.ai" or host.endswith(".opencode.ai"): + return ["--model-class", "opencode_model.OpenCodeModel"] + return [] + + +def _trajectory_summary(path: Path) -> dict: + if not path.exists(): + return {} + data = json.loads(path.read_text()) + info = data.get("info") or {} + stats = info.get("model_stats") or {} + messages = data.get("messages") or [] + tokens = {"prompt_tokens": 0, "completion_tokens": 0, "cached_prompt_tokens": 0} + cost, cost_calls = 0.0, 0 + for message in messages: + response = (message.get("extra") or {}).get("response") + usage = response.get("usage") if isinstance(response, dict) else None + if not isinstance(usage, dict): + continue + details = usage.get("prompt_tokens_details") or {} + tokens["prompt_tokens"] += int(usage.get("prompt_tokens") or 0) + tokens["completion_tokens"] += int(usage.get("completion_tokens") or 0) + tokens["cached_prompt_tokens"] += max( + int(usage.get("prompt_cache_hit_tokens") or 0), int(details.get("cached_tokens") or 0)) + if isinstance(usage.get("cost"), (int, float)) and not isinstance(usage.get("cost"), bool): + cost, cost_calls = cost + float(usage["cost"]), cost_calls + 1 + return { + "exit_status": info.get("exit_status"), + "model_calls": stats.get("api_calls"), + "messages": len(messages), + **tokens, + # Provider-reported spend (OpenRouter returns it in usage); None if never reported. + "cost_usd": round(cost, 6) if cost_calls else None, + "mini_swe_agent_version": info.get("mini_version"), + } + + +def run_baseline(rows: list[dict], *, run_dir: Path, run_name: str, settings: RunSettings, + grader: str, batch_size: int, workers: int, remove_images: bool) -> None: + api_key = os.environ.get(settings.api_key_env) + if not api_key: + raise SystemExit(f"set {settings.api_key_env} to the provider API key first") + executable = mini_executable() + mini_dir = run_dir / "mini" + data_dir = run_dir / "mini-data" + for directory in (mini_dir, data_dir, run_dir / "mini-config"): + directory.mkdir(parents=True, exist_ok=True) + # mini-swe-agent reads the image from image_name; Verified rows call it image. + (data_dir / "test.jsonl").write_text("".join( + json.dumps({**row, "image_name": row["image"]}, default=str) + "\n" for row in rows)) + overlay = run_dir / "mini-overlay.yaml" + overlay.write_text(yaml.safe_dump(overlay_config(settings), sort_keys=False)) + env = { + **os.environ, + "OPENAI_API_KEY": api_key, + "PYTHONPATH": str(MODEL_CLASS_DIR), + # Keep mini-swe-agent's global config out of the user's home directory. + "MSWEA_GLOBAL_CONFIG_DIR": str(run_dir / "mini-config"), + } + model_name = f"mini-swe-agent__{settings.model}" + pending = [row for row in rows + if not (run_dir / "instances" / row["instance_id"] / "prediction.json").exists()] + for start in range(0, len(pending), batch_size): + batch = pending[start:start + batch_size] + pattern = "^(" + "|".join(re.escape(row["instance_id"]) for row in batch) + ")$" + logger.info("mini-swe-agent batch %d-%d of %d", start + 1, start + len(batch), len(pending)) + with (run_dir / "mini.log").open("a") as log: + subprocess.run( + [str(executable), "swebench", "--subset", str(data_dir), "--split", "test", + "--filter", pattern, "-w", str(workers), "-o", str(mini_dir), + "-c", "swebench.yaml", "-c", str(overlay), + "-m", f"openai/{settings.model}", + *model_class_args(settings), + "--environment-class", "docker", + # Only pending tasks are in the filter; earlier outage attempts + # are still in preds.json and must be redone. + "--redo-existing"], + env=env, stdout=log, stderr=subprocess.STDOUT, check=False, + ) + predictions_file = mini_dir / "preds.json" + predictions = json.loads(predictions_file.read_text()) if predictions_file.exists() else {} + graded = [] + for row in batch: + instance_id = row["instance_id"] + instance_dir = run_dir / "instances" / instance_id + instance_dir.mkdir(parents=True, exist_ok=True) + patch = (predictions.get(instance_id) or {}).get("model_patch") or "" + summary = _trajectory_summary(mini_dir / instance_id / f"{instance_id}.traj.json") + if summary.get("exit_status") in PROVIDER_FAILURES: + (instance_dir / "infra_error.json").write_text(json.dumps({ + "reason": "provider_request_failed", "error_type": summary["exit_status"], + }, indent=2) + "\n") + continue + (instance_dir / "infra_error.json").unlink(missing_ok=True) + graded.append(row) + (instance_dir / "patch.diff").write_text(patch) + (instance_dir / "telemetry.json").write_text(json.dumps(summary, indent=2) + "\n") + (instance_dir / "result.json").write_text(json.dumps({ + "status": "completed" if instance_id in predictions else "error", + "stop_reason": summary.get("exit_status"), + }, indent=2) + "\n") + (instance_dir / "prediction.json").write_text(json.dumps({ + "instance_id": instance_id, "model_name_or_path": model_name, "model_patch": patch, + }) + "\n") + runner.write_predictions(run_dir) + for row in batch: + if row not in graded and remove_images: + docker_cli.remove_image(row["image"]) + for row in graded: + instance_dir = run_dir / "instances" / row["instance_id"] + patch = (instance_dir / "patch.diff").read_text() + result = evaluate.evaluate_instance( + run_dir=run_dir, run_id=run_name, model_name=model_name, + instance_id=row["instance_id"], patch=patch, grader=grader, + ) + (instance_dir / "eval.json").write_text(json.dumps(result, indent=2) + "\n") + logger.info("%s eval=%s", row["instance_id"], result["status"]) + if remove_images: + docker_cli.remove_image(row["image"]) + if batch and not graded: + logger.error("Stopping: every task in this batch failed on the model service. " + "Re-run the same command later; finished tasks are kept.") + break diff --git a/benchmarks/swebench/code4me_swebench/check.py b/benchmarks/swebench/code4me_swebench/check.py new file mode 100644 index 0000000..1273366 --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/check.py @@ -0,0 +1,141 @@ +"""Consistency checks over a run's telemetry traces. + +Each task's ``trace.jsonl`` must be one complete, ordered event stream: every +model request answered or recorded as failed, every tool call resolved, token +metrics equal to the provider's own usage, and no provider key anywhere. +""" + +from __future__ import annotations + +import json +import os +import re +from collections import Counter +from datetime import datetime +from typing import TYPE_CHECKING + +from code4me_swebench.settings import pinned_upstreams +from code4me_swebench.telemetry import read_events, upstream + +if TYPE_CHECKING: + from pathlib import Path + +SCHEMA = "code4me.agent.event.v1" +TOOL_OUTCOMES = {"agent.tool.completed", "agent.tool.failed", "agent.tool.denied"} + + +def _time(event: dict) -> datetime: + return datetime.fromisoformat(event["timestamp"].replace("Z", "+00:00")) + + +def _upstream_key(name: str) -> str: + # "deepseek" (slug) ~ "DeepSeek" (response); "deepinfra/fp8" ~ "DeepInfra". + return re.sub(r"[^a-z0-9]", "", name.split("/")[0].lower()) + + +def check_trace(trace_path: Path, *, secret: str | None = None, + upstreams: list[str] | None = None) -> list[str]: + """Problems found in one trace (empty = consistent).""" + if not trace_path.exists(): + return ["trace.jsonl missing"] + raw = trace_path.read_bytes() + problems: list[str] = [] + if secret and secret.encode() in raw: + problems.append("provider key found in trace") + try: + events = read_events(trace_path) + except json.JSONDecodeError as exc: + return [*problems, f"unparseable line: {exc}"] + if not events: + return [*problems, "trace is empty"] + + kinds = Counter(event.get("event_type") for event in events) + if any(event.get("schema_version") != SCHEMA for event in events): + problems.append("event with another schema_version") + for field in ("run_id", "session_id"): + values = {event.get(field) for event in events} + if len(values) != 1: + problems.append(f"{len(values)} distinct {field} values") + if [event.get("sequence") for event in events] != list(range(1, len(events) + 1)): + problems.append("sequence numbers are not 1..N in file order") + if any(_time(b) < _time(a) for a, b in zip(events, events[1:])): + problems.append("timestamps go backwards") + if events[0].get("event_type") != "agent.run.started": + problems.append("first event is not agent.run.started") + if kinds["agent.run.completed"] != 1: + problems.append(f"{kinds['agent.run.completed']} agent.run.completed events") + + # A request is answered by agent.model.completed, by a main-loop failure + # (agent.adapter.loop_failed for the "turn" call), or by a failed optional + # side call counted on agent.run.completed (metrics.failed_side_calls). + loop_failures = sum(1 for e in events if e["event_type"] == "agent.adapter.loop_failed" + and (e.get("payload") or {}).get("call_purpose") == "turn") + side_failures = sum(int((e.get("metrics") or {}).get("failed_side_calls") or 0) + for e in events if e["event_type"] == "agent.run.completed") + answered = kinds["agent.model.completed"] + loop_failures + side_failures + if kinds["agent.model.requested"] != answered: + problems.append( + f"{kinds['agent.model.requested']} model requests but {answered} completed or failed") + + called = {e["payload"].get("tool_call_id") for e in events if e["event_type"] == "agent.tool.called"} + resolved = {e["payload"].get("tool_call_id") for e in events if e["event_type"] in TOOL_OUTCOMES} + if called - resolved: + problems.append(f"{len(called - resolved)} tool calls without an outcome") + + ids = {event.get("event_id") for event in events} + dangling = sum(1 for e in events if e.get("parent_event_id") and e["parent_event_id"] not in ids) + if dangling: + problems.append(f"{dangling} parent_event_id values point nowhere") + + mismatched = 0 + for event in events: + if event["event_type"] != "agent.model.completed" or event["payload"].get("usage_estimated"): + continue + usage = ((event.get("raw_payload") or {}).get("response") or {}).get("usage") + if not isinstance(usage, dict): + problems.append("model.completed without raw provider usage") + break + metrics = event.get("metrics") or {} + if (metrics.get("prompt_tokens"), metrics.get("completion_tokens")) != ( + usage.get("prompt_tokens"), usage.get("completion_tokens")): + mismatched += 1 + if mismatched: + problems.append(f"{mismatched} model calls whose token metrics differ from provider usage") + + # A pinned upstream (OpenRouter): every call must name one of the pinned hosts. + if upstreams: + allowed = {_upstream_key(slug) for slug in upstreams} + served = Counter(upstream(e) for e in events if e["event_type"] == "agent.model.completed") + foreign = {name: n for name, n in served.items() if name and _upstream_key(name) not in allowed} + unnamed = served.get(None, 0) + if foreign: + problems.append(f"model calls served by unpinned upstreams: {foreign}") + if unnamed: + problems.append(f"{unnamed} model calls without the serving upstream in the response") + return problems + + +def check_run(run_dir: Path, *, secret_env: str | None = None) -> dict: + manifest = json.loads((run_dir / "manifest.json").read_text()) + secret = os.environ.get(secret_env or manifest.get("settings", {}).get("api_key_env") or "") + if "baseline" in manifest: + return {"run": run_dir.name, "skipped": "baseline run (mini-swe-agent trajectories, no agent trace)"} + results: dict[str, list[str]] = {} + for instance_id in manifest["instance_ids"]: + directory = run_dir / "instances" / instance_id + if not (directory / "prediction.json").exists(): + results[instance_id] = ["not run"] + continue + problems = check_trace(directory / "trace.jsonl", secret=secret or None, + upstreams=pinned_upstreams(manifest.get("settings", {}).get("extra_body") or {})) + result_file = directory / "result.json" + if not result_file.exists() or json.loads(result_file.read_text()).get("status") != "completed": + problems.append("driver did not complete (result.json)") + results[instance_id] = problems + return { + "run": run_dir.name, + "tasks": len(results), + "consistent": sum(1 for problems in results.values() if not problems), + "key_checked": bool(secret), + "problems": {iid: problems for iid, problems in results.items() if problems}, + } diff --git a/benchmarks/swebench/code4me_swebench/cli.py b/benchmarks/swebench/code4me_swebench/cli.py new file mode 100644 index 0000000..182d73f --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/cli.py @@ -0,0 +1,358 @@ +"""code4me-swebench: run the Code4Me agent on SWE-bench Verified tasks.""" + +from __future__ import annotations + +import argparse +import json +import logging +from pathlib import Path + +from code4me_swebench import dataset, report, runner +from code4me_swebench.settings import ( + RunSettings, + is_openrouter, + openrouter_routing, + pinned_upstreams, +) + +DEFAULT_RUNS_DIR = Path(__file__).resolve().parents[1] / "runs" + + +def _extra_body(args: argparse.Namespace) -> dict: + extra = json.loads(args.extra_body) if getattr(args, "extra_body", None) else {} + if not isinstance(extra, dict): + raise SystemExit("--extra-body must be a JSON object") + if getattr(args, "openrouter_upstream", None): + extra = {**extra, **openrouter_routing(args.openrouter_upstream)} + if is_openrouter(args.base_url): + if not pinned_upstreams(extra): + # OpenRouter serves one model from many hosts (fp4/fp8/full precision, + # different limits) and may switch per request: results would mix them. + raise SystemExit("OpenRouter can route each request to a different upstream; pin one with " + "--openrouter-upstream SLUG (e.g. deepseek)") + # DeepSeek V4 / V4.1 Flash think at effort "high" by default, and the model + # card recommends top_p 1.0 (with temperature 1.0, see _settings): stated + # explicitly (and so recorded), as hosts differ in their own defaults. + extra.setdefault("reasoning", {"effort": "high"}) + extra.setdefault("top_p", 1.0) + return extra + + +def _temperature(args: argparse.Namespace) -> float | None: + if args.temperature is None and is_openrouter(args.base_url): + return 1.0 # the model card's (and the SWE-rebench leaderboard's) setting + return args.temperature + + +def _settings(args: argparse.Namespace) -> RunSettings: + return RunSettings( + model=args.model, + base_url=args.base_url, + api_key_env=args.api_key_env, + temperature=_temperature(args), + max_iterations=args.max_iterations, + context_tokens=args.context_tokens, + task_timeout_s=args.task_timeout, + platform=args.platform, + # Explicit and recorded in the manifest: runs differ from the shipped + # defaults only where a flag says so. + harness_options={"self_review": False} if getattr(args, "no_self_review", False) else {}, + extra_body=_extra_body(args), + ) + + +def cmd_select(args: argparse.Namespace) -> None: + rows = dataset.load_rows(args.dataset, args.split) + if args.window: + start, end = args.window + ids = dataset.in_window(rows, start, end) + revision = dataset.default_revision(args.dataset) + how = (f"every task created in [{start}, {end}) (a SWE-rebench leaderboard window)" + + (f", dataset revision {revision}" if revision else "")) + header = (f"# {len(ids)} instances of {args.dataset} ({args.split}), {how}. " + "Generated by `code4me-swebench select --window`.\n") + Path(args.out).write_text(header + "".join(f"{iid}\n" for iid in ids)) + print(f"wrote {len(ids)} ids to {args.out}") + return + if args.newest: + ids = dataset.newest(rows, args.count) + created = sorted(str(row.get("created_at")) for row in rows if row["instance_id"] in set(ids)) + how = f"the newest by created_at ({created[0][:10]} .. {created[-1][:10]})" + else: + ids = dataset.stratified_sample(rows, args.count, seed=args.seed) + how = f"stratified by repo x difficulty (largest remainder), seed {args.seed}" + header = (f"# {args.count} instances of {args.dataset} ({args.split}), {how}. " + "Generated by `code4me-swebench select`.\n") + Path(args.out).write_text(header + "".join(f"{iid}\n" for iid in ids)) + print(f"wrote {len(ids)} ids to {args.out}") + + +def cmd_runtime(args: argparse.Namespace) -> None: + from code4me_swebench.runtime import ensure_runtime + + print(json.dumps(ensure_runtime(platform=args.platform, rebuild=args.rebuild).as_dict(), indent=2)) + + +def repeat_names(run_name: str, repeats: int) -> list[str]: + return [run_name] if repeats <= 1 else [f"{run_name}-r{index}" for index in range(1, repeats + 1)] + + +def cmd_run(args: argparse.Namespace) -> None: + from code4me_swebench.runtime import ensure_runtime + + ids = args.instances or dataset.read_subset(dataset.resolve_subset(args.subset)) + if args.limit: + ids = ids[: args.limit] + rows = dataset.select_rows(dataset.load_rows(args.dataset, args.split), ids) + grader = args.grader or dataset.default_grader(args.dataset) + settings = _settings(args) + runtime = ensure_runtime(platform=settings.platform) + runs = [] + for name in repeat_names(args.run_name, args.repeats): + run_dir = args.runs_dir / name + runner.prepare_run(run_dir, rows=rows, settings=settings, runtime=runtime, + subset=args.subset if not args.instances else "explicit", + dataset_name=args.dataset, split=args.split, grader=grader, + dataset_revision=dataset.default_revision(args.dataset)) + runs.append((run_dir, name)) + options = runner.RunOptions( + evaluate=not args.no_eval, remove_images=args.remove_images, force=args.force, + memory=args.memory, cpus=args.cpus, grader=grader, + ) + runner.run_repeated(rows, runs=runs, settings=settings, runtime=runtime, + options=options, workers=args.workers) + for run_dir, _name in runs: + _print_report(run_dir) + if len(runs) > 1: + print(json.dumps(report.aggregate_runs([run_dir for run_dir, _ in runs]), indent=2)) + + +def cmd_baseline_mini(args: argparse.Namespace) -> None: + from code4me_swebench import baseline_mini + + ids = args.instances or dataset.read_subset(dataset.resolve_subset(args.subset)) + if args.limit: + ids = ids[: args.limit] + rows = dataset.select_rows(dataset.load_rows(args.dataset, args.split), ids) + grader = dataset.default_grader(args.dataset) + settings = _settings(args) + run_dir = args.runs_dir / args.run_name + run_dir.mkdir(parents=True, exist_ok=True) + manifest = { + "subset": args.subset if not args.instances else "explicit", + "dataset": args.dataset, "split": args.split, "grader": grader, + "instance_ids": ids, + "settings": {"model": settings.model, "base_url": settings.base_url, + "temperature": settings.temperature, "platform": settings.platform, + **({"extra_body": settings.extra_body} if settings.extra_body else {})}, + "baseline": {"agent": "mini-swe-agent", "version": baseline_mini.MINI_VERSION, + "config": "stock swebench.yaml + mini-overlay.yaml", + "network": "none (model called from the host)"}, + } + manifest_path = run_dir / "manifest.json" + if manifest_path.exists() and json.loads(manifest_path.read_text()) != manifest: + raise SystemExit(f"{run_dir} was started with other settings; use a new --run-name") + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + (run_dir / "dataset.json").write_text(json.dumps(rows, default=str)) + baseline_mini.run_baseline(rows, run_dir=run_dir, run_name=args.run_name, settings=settings, + grader=grader, batch_size=args.batch_size, workers=args.workers, + remove_images=args.remove_images) + _print_report(run_dir) + + +def cmd_evaluate(args: argparse.Namespace) -> None: + from code4me_swebench import evaluate + + run_dir = args.runs_dir / args.run_name + manifest = json.loads((run_dir / "manifest.json").read_text()) + model_name = f"code4me2-agent__{manifest['settings']['model']}" + for instance_id in manifest["instance_ids"]: + directory = run_dir / "instances" / instance_id + if not (directory / "prediction.json").exists(): + continue + if (directory / "eval.json").exists() and not args.force: + continue + patch = json.loads((directory / "prediction.json").read_text())["model_patch"] + result = evaluate.evaluate_instance( + run_dir=run_dir, run_id=args.run_name, model_name=model_name, + instance_id=instance_id, patch=patch, grader=manifest.get("grader", "swebench"), + ) + (directory / "eval.json").write_text(json.dumps(result, indent=2) + "\n") + print(instance_id, result["status"]) + _print_report(run_dir) + + +def cmd_report(args: argparse.Namespace) -> None: + _print_report(args.runs_dir / args.run_name) + + +def cmd_requeue(args: argparse.Namespace) -> None: + released = runner.requeue_outages(args.runs_dir / args.run_name) + print(f"released {len(released)} task(s) for a re-run: {released}") + + +def cmd_check(args: argparse.Namespace) -> None: + from code4me_swebench import check + + names = args.run_names or sorted(p.name for p in args.runs_dir.iterdir() if (p / "manifest.json").exists()) + results = [check.check_run(args.runs_dir / name) for name in names] + print(json.dumps(results, indent=2)) + if any(result.get("problems") for result in results): + raise SystemExit(1) + + +# The SWE-rebench leaderboard windows inside HF split 2026_03 (half-month aligned). +REBENCH_2026_03_WINDOWS = ( + ("2026-03-01", "2026-05-15"), ("2026-03-01", "2026-04-01"), ("2026-04-01", "2026-05-15"), + ("2026-04-15", "2026-05-15"), ("2026-05-01", "2026-05-15"), +) + + +def cmd_aggregate(args: argparse.Namespace) -> None: + names = args.runs or repeat_names(args.run_name, args.repeats) + if not names: + raise SystemExit("name the runs, or give --run-name and --repeats") + windows = [tuple(window) for window in args.window] if args.window else list(REBENCH_2026_03_WINDOWS) + result = report.aggregate_runs([args.runs_dir / name for name in names], windows=windows, + reference=args.reference) + out = args.runs_dir / f"{args.run_name or names[0]}-aggregate.json" + out.write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result, indent=2)) + print(f"aggregate: {out}") + + +def cmd_compare(args: argparse.Namespace) -> None: + print(json.dumps(report.compare_runs(args.runs_dir / args.a, args.runs_dir / args.b), indent=2)) + + +def _print_report(run_dir: Path) -> None: + data = report.write_report(run_dir) + keep = ("run", "model", "subset", "instances", "graded", "resolved", "resolve_rate", + "resolve_rate_95ci", "agent_status", "eval_status", "model_calls", + "prompt_tokens", "cached_prompt_tokens", "completion_tokens", "agent_wall_s") + print(json.dumps({key: data[key] for key in keep}, indent=2)) + print(f"report: {run_dir / 'report.json'} table: {run_dir / 'instances.csv'}") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="code4me-swebench", description=__doc__) + parser.add_argument("--runs-dir", type=Path, default=DEFAULT_RUNS_DIR) + parser.add_argument("--dataset", default=dataset.DATASET, + help=f"{dataset.DATASET} or {dataset.REBENCH_DATASET}") + parser.add_argument("--split", default="test", + help="dataset split; SWE-rebench uses monthly splits such as 2026_03") + parser.add_argument("--platform", default="linux/amd64", + help="SWE-bench images are published for linux/amd64 only.") + sub = parser.add_subparsers(dest="command", required=True) + + select = sub.add_parser("select", help="write a stratified subset file") + select.add_argument("--count", type=int, default=50) + select.add_argument("--seed", type=int, default=0) + select.add_argument("--newest", action="store_true", + help="take the most recently created tasks instead of a stratified sample") + select.add_argument("--window", nargs=2, metavar=("START", "END"), + help="every task created in [START, END), e.g. 2026-04-01 2026-05-15") + select.add_argument("--out", required=True) + select.set_defaults(func=cmd_select) + + runtime = sub.add_parser("runtime", help="build the agent runtime volume") + runtime.add_argument("--rebuild", action="store_true") + runtime.set_defaults(func=cmd_runtime) + + run = sub.add_parser("run", help="run the agent on a subset (resumable)") + run.add_argument("--run-name", required=True) + run.add_argument("--subset", default="verified-mini", + help="bundled subset name (subsets/*.txt) or a path to an id list") + run.add_argument("--instances", nargs="+", help="explicit instance ids instead of a subset") + run.add_argument("--limit", type=int, help="only the first N instances of the subset") + defaults = RunSettings() + run.add_argument("--model", default=defaults.model) + run.add_argument("--base-url", default=defaults.base_url) + run.add_argument("--api-key-env", default=defaults.api_key_env) + run.add_argument("--temperature", type=float, default=defaults.temperature) + run.add_argument("--max-iterations", type=int, default=defaults.max_iterations) + run.add_argument("--context-tokens", type=int, default=defaults.context_tokens) + run.add_argument("--task-timeout", type=float, default=defaults.task_timeout_s) + run.add_argument("--no-self-review", action="store_true", + help="switch off the agent's end-of-task self-review (harness_options.self_review)") + run.add_argument("--workers", type=int, default=1) + run.add_argument("--memory", help="docker --memory for task containers, e.g. 4g") + run.add_argument("--cpus", help="docker --cpus for task containers, e.g. 2") + run.add_argument("--no-eval", action="store_true", help="skip grading after each task") + run.add_argument("--remove-images", action="store_true", + help="delete each task image after grading (bounded disk use)") + run.add_argument("--force", action="store_true", help="re-run tasks that already finished") + run.add_argument("--repeats", type=int, default=1, + help="independent runs of every task (RUN_NAME-r1..rN), task by task so each " + "image is pulled once; the SWE-rebench leaderboard uses 5") + run.add_argument("--grader", choices=("swebench", "rebench"), + help="default: rebench for SWE-rebench datasets, else swebench") + run.set_defaults(func=cmd_run) + + mini = sub.add_parser("baseline-mini", help="run the mini-swe-agent baseline on a subset") + mini.add_argument("--run-name", required=True) + mini.add_argument("--subset", default="verified-mini") + mini.add_argument("--instances", nargs="+") + mini.add_argument("--limit", type=int) + mini.add_argument("--model", default=defaults.model) + mini.add_argument("--base-url", default=defaults.base_url) + mini.add_argument("--api-key-env", default=defaults.api_key_env) + mini.add_argument("--temperature", type=float, default=defaults.temperature) + mini.add_argument("--workers", type=int, default=1) + mini.add_argument("--batch-size", type=int, default=4, + help="tasks per mini-swe-agent call; images are deleted after each batch") + mini.add_argument("--remove-images", action="store_true") + mini.set_defaults(func=cmd_baseline_mini, max_iterations=defaults.max_iterations, + context_tokens=defaults.context_tokens, task_timeout=defaults.task_timeout_s) + # Both agents must reach the model the same way for a paired comparison. + for command in (run, mini): + command.add_argument("--openrouter-upstream", action="append", metavar="SLUG", + help="serve every request from this OpenRouter upstream, e.g. deepseek " + "(required with an openrouter.ai --base-url)") + command.add_argument("--extra-body", metavar="JSON", + help="extra top-level request fields for the model API, as a JSON object") + + grade = sub.add_parser("evaluate", help="grade finished tasks that have no result yet") + grade.add_argument("--run-name", required=True) + grade.add_argument("--force", action="store_true") + grade.set_defaults(func=cmd_evaluate) + + req = sub.add_parser("requeue", help="release tasks whose attempt failed on the model service") + req.add_argument("--run-name", required=True) + req.set_defaults(func=cmd_requeue) + + chk = sub.add_parser("check", help="validate the telemetry traces of runs (default: all)") + chk.add_argument("run_names", nargs="*") + chk.set_defaults(func=cmd_check) + + cmp_ = sub.add_parser("compare", help="paired comparison of two runs on their shared tasks") + cmp_.add_argument("a", help="run name, e.g. our agent") + cmp_.add_argument("b", help="run name, e.g. the mini-swe-agent baseline") + cmp_.set_defaults(func=cmd_compare) + + agg = sub.add_parser("aggregate", help="leaderboard metrics over repeated runs (mean, SEM, pass@n)") + agg.add_argument("runs", nargs="*", help="run names (default: RUN_NAME-r1..rN)") + agg.add_argument("--run-name") + agg.add_argument("--repeats", type=int, default=5) + agg.add_argument("--window", nargs=2, action="append", metavar=("START", "END"), + help="also report tasks created in [START, END); default: the 2026_03 leaderboard windows") + agg.add_argument("--reference", help="published result to compare with (references.json key), " + "e.g. deepseek-v4-flash-high") + agg.set_defaults(func=cmd_aggregate) + + rep = sub.add_parser("report", help="rebuild report.json and instances.csv") + rep.add_argument("--run-name", required=True) + rep.set_defaults(func=cmd_report) + return parser + + +def main(argv: list[str] | None = None) -> None: + # Third-party libraries (HF hub, httpx) stay at WARNING; this package at INFO. + logging.basicConfig(level=logging.WARNING, format="%(asctime)s %(levelname)s %(message)s") + logging.getLogger("code4me_swebench").setLevel(logging.INFO) + args = build_parser().parse_args(argv) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/swebench/code4me_swebench/dataset.py b/benchmarks/swebench/code4me_swebench/dataset.py new file mode 100644 index 0000000..ee9e5b1 --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/dataset.py @@ -0,0 +1,108 @@ +"""Task selection from SWE-bench Verified. + +``SWE-bench/SWE-bench_Verified`` is used (not ``princeton-nlp/...``) because it +carries the per-instance ``image`` and ``eval_script`` that swebench 5 grades with. +""" + +from __future__ import annotations + +import random +from collections import defaultdict +from pathlib import Path + +DATASET = "SWE-bench/SWE-bench_Verified" +REBENCH_DATASET = "nebius/SWE-rebench-leaderboard" +# The dataset revision whose split 2026_03 is exactly the leaderboard window +# 2026-03-01 -> 2026-05-15 (110 tasks; checked 2026-10-14). Pinned so a later +# fix to the dataset cannot silently change the task set between runs. +REBENCH_REVISION = "34d5a58864acf91613740a09ec5d205228dcfa39" +SUBSETS_DIR = Path(__file__).resolve().parents[1] / "subsets" + + +def default_revision(dataset: str) -> str | None: + return REBENCH_REVISION if dataset == REBENCH_DATASET else None + + +def load_rows(dataset: str = DATASET, split: str = "test", revision: str | None = None) -> list[dict]: + from datasets import load_dataset + + rows = load_dataset(dataset, split=split, revision=revision or default_revision(dataset)) + return [normalize_row(dict(row)) for row in rows] + + +def in_window(rows: list[dict], start: str, end: str) -> list[str]: + """Ids created in [start, end) (YYYY-MM-DD), oldest first: a leaderboard time + window. SWE-rebench windows are half-month aligned and end-exclusive.""" + chosen = [row for row in rows if start <= str(row.get("created_at"))[:10] < end] + return [row["instance_id"] for row in sorted(chosen, key=lambda r: (str(r.get("created_at")), r["instance_id"]))] + + +def normalize_row(row: dict) -> dict: + """Give every row an ``image`` (SWE-rebench names it image_name/docker_image).""" + if not row.get("image"): + image = row.get("image_name") or row.get("docker_image") + if image: + row["image"] = image + return row + + +def default_grader(dataset_name: str) -> str: + return "rebench" if "rebench" in dataset_name.lower() else "swebench" + + +def read_subset(path: Path) -> list[str]: + ids = [] + for line in path.read_text().splitlines(): + line = line.split("#", 1)[0].strip() + if line: + ids.append(line) + if len(ids) != len(set(ids)): + raise ValueError(f"{path} lists an instance twice") + return ids + + +def resolve_subset(name_or_path: str) -> Path: + path = Path(name_or_path) + if path.exists(): + return path + bundled = SUBSETS_DIR / f"{name_or_path}.txt" + if bundled.exists(): + return bundled + raise FileNotFoundError(f"no subset file {name_or_path!r} (bundled: {sorted(p.stem for p in SUBSETS_DIR.glob('*.txt'))})") + + +def select_rows(rows: list[dict], ids: list[str]) -> list[dict]: + by_id = {row["instance_id"]: row for row in rows} + missing = [iid for iid in ids if iid not in by_id] + if missing: + raise KeyError(f"instances not in the dataset: {missing}") + return [by_id[iid] for iid in ids] + + +def stratified_sample(rows: list[dict], count: int, *, seed: int) -> list[str]: + """Pick ``count`` ids proportionally over (repo, difficulty) cells. + + Allocation uses the largest-remainder method, so every cell gets its + proportional share rounded fairly; ties break by cell name for determinism. + """ + cells: dict[tuple[str, str], list[str]] = defaultdict(list) + for row in rows: + cells[(row["repo"], row.get("difficulty") or "unknown")].append(row["instance_id"]) + total = len(rows) + quotas = {cell: len(ids) * count / total for cell, ids in cells.items()} + allocation = {cell: int(quota) for cell, quota in quotas.items()} + remaining = count - sum(allocation.values()) + by_remainder = sorted(quotas, key=lambda cell: (-(quotas[cell] - allocation[cell]), cell)) + for cell in by_remainder[:remaining]: + allocation[cell] += 1 + rng = random.Random(seed) + chosen: list[str] = [] + for cell in sorted(cells): + chosen.extend(rng.sample(sorted(cells[cell]), allocation[cell])) + return sorted(chosen) + + +def newest(rows: list[dict], count: int) -> list[str]: + """The ``count`` most recently created tasks (least likely to be in training data).""" + ordered = sorted(rows, key=lambda row: (str(row.get("created_at") or ""), row["instance_id"])) + return sorted(row["instance_id"] for row in ordered[-count:]) diff --git a/benchmarks/swebench/code4me_swebench/docker_cli.py b/benchmarks/swebench/code4me_swebench/docker_cli.py new file mode 100644 index 0000000..dabee12 --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/docker_cli.py @@ -0,0 +1,107 @@ +"""Thin wrappers over the ``docker`` CLI (the same on macOS/Colima and Linux).""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + + +class DockerError(RuntimeError): + pass + + +@dataclass(frozen=True) +class Completed: + returncode: int + stdout: str + stderr: str + timed_out: bool = False + + +def docker(*args: str, check: bool = True, timeout: float | None = None, + env: dict[str, str] | None = None) -> Completed: + try: + proc = subprocess.run( + ["docker", *args], capture_output=True, text=True, errors="replace", + timeout=timeout, env=env, + ) + except subprocess.TimeoutExpired as exc: + if check: + raise DockerError(f"docker {' '.join(args[:3])} timed out after {timeout} s") from None + return Completed(-1, _text(exc.stdout), _text(exc.stderr), timed_out=True) + result = Completed(proc.returncode, proc.stdout, proc.stderr) + if check and proc.returncode != 0: + raise DockerError(f"docker {' '.join(args[:3])} failed: {proc.stderr.strip()[-2000:]}") + return result + + +def _text(value: bytes | str | None) -> str: + if value is None: + return "" + return value.decode("utf-8", "replace") if isinstance(value, bytes) else value + + +def image_present(image: str) -> bool: + return docker("image", "inspect", image, check=False).returncode == 0 + + +def pull(image: str, *, platform: str) -> None: + if not image_present(image): + docker("pull", "--platform", platform, "--quiet", image) + + +def volume_exists(name: str) -> bool: + return docker("volume", "inspect", name, check=False).returncode == 0 + + +def start_container(*, name: str, image: str, platform: str, mounts: list[str], + memory: str | None = None, cpus: str | None = None, + network: str | None = None) -> None: + docker("rm", "-f", name, check=False) + args = ["run", "-d", "--name", name, "--platform", platform, "--entrypoint", ""] + if network: + args += ["--network", network] + for mount in mounts: + args += ["-v", mount] + if memory: + args += ["--memory", memory] + if cpus: + args += ["--cpus", cpus] + docker(*args, image, "sleep", "infinity") + + +def exec_in(container: str, *argv: str, workdir: str | None = None, + env: list[str] | None = None, timeout: float | None = None, + check: bool = False, process_env: dict[str, str] | None = None) -> Completed: + args = ["exec"] + if workdir: + args += ["-w", workdir] + for item in env or []: + args += ["-e", item] + return docker(*args, container, *argv, check=check, timeout=timeout, env=process_env) + + +def copy_to(container: str, source: Path, destination: str) -> None: + docker("cp", str(source), f"{container}:{destination}") + + +def copy_from(container: str, source: str, destination: Path) -> bool: + destination.mkdir(parents=True, exist_ok=True) + return docker("cp", f"{container}:{source}", str(destination), check=False).returncode == 0 + + +def remove_container(name: str) -> None: + docker("rm", "-f", name, check=False) + + +def remove_image(image: str) -> None: + docker("image", "rm", image, check=False) + + +def container_ip(name: str, network: str) -> str: + template = '{{(index .NetworkSettings.Networks "%s").IPAddress}}' % network + return docker("inspect", "--format", template, name, check=False).stdout.strip() diff --git a/benchmarks/swebench/code4me_swebench/driver.py b/benchmarks/swebench/code4me_swebench/driver.py new file mode 100644 index 0000000..be404d8 --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/driver.py @@ -0,0 +1,79 @@ +"""Runs one SWE-bench task with the Code4Me agent, inside the task container. + +This file is copied into the container and executed by the runtime's Python +(``/opt/code4me/venv``), so it may import only the standard library and +``code4me2_agent``. It drives the agent core directly: no IDE, plugin or ACP +transport. Telemetry is the agent's own canonical event stream +(``code4me.agent.event.v1``), written as JSONL to ``trace_path``. + +Usage: python driver.py +""" + +from __future__ import annotations + +import json +import logging +import os +import sys +import traceback +from dataclasses import replace +from pathlib import Path +from time import perf_counter + + +def build_agent_config(task: dict): + from code4me2_agent.config import AgentConfig + + out_dir = Path(task["out_dir"]) + out_dir.mkdir(parents=True, exist_ok=True) + config_path = out_dir / "agent-config.json" + config_path.write_text(json.dumps(task["agent_config"], indent=2) + "\n") + config = AgentConfig.from_file(config_path) + # The config file format has no tool/approval fields: those normally come + # from the assigned profile, so they are applied here the same way. + allowed = task.get("allowed_tools") + return replace( + config, + tools=list(allowed) if allowed is not None else None, + allowed_tools=frozenset(allowed) if allowed is not None else None, + approval_policy="auto", + store_agent_content=True, + ) + + +def main(argv: list[str]) -> int: + # The agent logs provider retries and harness decisions; they land in agent.log. + logging.basicConfig( + level=getattr(logging, os.environ.get("CODE4ME_AGENT_LOG_LEVEL", "INFO").upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + task = json.loads(Path(argv[1]).read_text()) + out_dir = Path(task["out_dir"]) + out_dir.mkdir(parents=True, exist_ok=True) + result: dict = {"instance_id": task["instance_id"], "status": "error"} + started = perf_counter() + try: + from code4me2_agent._build_version import RUNTIME_VERSION + from code4me2_agent.echo import EchoAgentCore + + result["runtime_version"] = RUNTIME_VERSION + core = EchoAgentCore(build_agent_config(task)) + outcome = core.handle_prompt(task["prompt"], run_id=task["run_id"]) + result.update( + status="completed", + run_id=outcome.run_id, + stop_reason=outcome.stop_reason, + run_status=outcome.run_status, + final_response=outcome.final_response, + usage=outcome.usage, + ) + except BaseException as exc: # noqa: BLE001 - recorded for the report + result["error"] = f"{type(exc).__name__}: {exc}" + result["traceback"] = traceback.format_exc() + result["duration_s"] = round(perf_counter() - started, 3) + (out_dir / "result.json").write_text(json.dumps(result, indent=2) + "\n") + return 0 if result["status"] == "completed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/benchmarks/swebench/code4me_swebench/egress_proxy.py b/benchmarks/swebench/code4me_swebench/egress_proxy.py new file mode 100644 index 0000000..98106b6 --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/egress_proxy.py @@ -0,0 +1,98 @@ +"""Allowlist-only HTTPS egress proxy for benchmark task containers. + +Task containers sit on an internal Docker network with no route out; this proxy +is their only exit and tunnels (HTTP CONNECT) to the allowlisted hosts only, so +the agent can reach its model API but not GitHub, PyPI or anything else that +could hold the upstream fix. Every decision is logged for the audit trail. + +Standalone and stdlib-only: copied into a ``python:3.12-alpine`` container. +Environment: ALLOW_HOSTS (comma-separated; a host also allows its subdomains), +ALLOW_PORTS (default 443), PROXY_PORT (default 3128). +""" + +from __future__ import annotations + +import asyncio +import os +import sys +from datetime import datetime, timezone + + +def parse_hosts(value: str) -> frozenset[str]: + return frozenset(h.strip().lower().rstrip(".") for h in value.split(",") if h.strip()) + + +def is_allowed(host: str, port: int, *, hosts: frozenset[str], ports: frozenset[int]) -> bool: + host = host.strip().lower().rstrip(".") + if port not in ports or not host: + return False + return any(host == allowed or host.endswith("." + allowed) for allowed in hosts) + + +def parse_connect(request_line: str) -> tuple[str, int] | None: + """``CONNECT host:port HTTP/1.1`` → (host, port); anything else → None.""" + parts = request_line.split() + if len(parts) != 3 or parts[0].upper() != "CONNECT": + return None + host, sep, port = parts[1].rpartition(":") + if not sep or not port.isdigit(): + return None + return host.strip("[]"), int(port) + + +def _log(decision: str, peer: str, target: str) -> None: + stamp = datetime.now(timezone.utc).isoformat(timespec="seconds") + print(f"{stamp} {decision} {peer} {target}", flush=True) + + +async def _pipe(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + while data := await reader.read(65536): + writer.write(data) + await writer.drain() + except (ConnectionError, OSError): + pass + finally: + writer.close() + + +async def _handle(client_r, client_w, *, hosts, ports) -> None: + peer = str((client_w.get_extra_info("peername") or ("?",))[0]) + try: + head = await asyncio.wait_for(client_r.readuntil(b"\r\n\r\n"), timeout=30) + request_line = head.split(b"\r\n", 1)[0].decode("latin-1") + target = parse_connect(request_line) + if target is None or not is_allowed(*target, hosts=hosts, ports=ports): + _log("DENY", peer, request_line[:200]) + client_w.write(b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n") + await client_w.drain() + client_w.close() + return + upstream_r, upstream_w = await asyncio.wait_for(asyncio.open_connection(*target), timeout=30) + _log("ALLOW", peer, f"{target[0]}:{target[1]}") + client_w.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + await client_w.drain() + await asyncio.gather(_pipe(client_r, upstream_w), _pipe(upstream_r, client_w)) + except (asyncio.TimeoutError, asyncio.IncompleteReadError, ConnectionError, OSError, ValueError): + client_w.close() + + +async def serve(host: str, port: int, *, hosts: frozenset[str], ports: frozenset[int]) -> asyncio.Server: + return await asyncio.start_server( + lambda r, w: _handle(r, w, hosts=hosts, ports=ports), host, port) + + +async def _main() -> None: + hosts = parse_hosts(os.environ.get("ALLOW_HOSTS", "")) + ports = frozenset(int(p) for p in os.environ.get("ALLOW_PORTS", "443").split(",") if p.strip()) + port = int(os.environ.get("PROXY_PORT", "3128")) + if not hosts: + sys.exit("ALLOW_HOSTS is empty: refusing to start an allow-nothing proxy by accident") + server = await serve("0.0.0.0", port, hosts=hosts, ports=ports) + _log("START", "-", f"allow={','.join(sorted(hosts))} ports={','.join(map(str, sorted(ports)))}") + async with server: + await server.serve_forever() + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/benchmarks/swebench/code4me_swebench/evaluate.py b/benchmarks/swebench/code4me_swebench/evaluate.py new file mode 100644 index 0000000..667571d --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/evaluate.py @@ -0,0 +1,104 @@ +"""Grading with the official SWE-bench harness (``swebench.harness.run_evaluation``).""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import threading +from pathlib import Path + + +def docker_env() -> dict[str, str]: + """Process env whose DOCKER_HOST matches the docker CLI's current context. + + The harness uses the Docker Python SDK, which ignores CLI contexts (Colima + serves its socket under ~/.colima, not /var/run/docker.sock). + """ + env = dict(os.environ) + if env.get("DOCKER_HOST"): + return env + host = subprocess.run( + ["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], + capture_output=True, text=True, + ).stdout.strip() + if host: + env["DOCKER_HOST"] = host + return env + + +# SWE-rebench grades with its own fork of the SWE-bench harness (swebench 4.0.3, +# which conflicts with our swebench 5), so it lives in a separate venv. +REBENCH_HARNESS = ( + "swebench @ git+https://github.com/SWE-rebench/SWE-bench-fork" + "@e4907b7a90eafaa1f0a6428fd04fe31cdd8b4284" +) +REBENCH_VENV = Path(__file__).resolve().parents[1] / ".venv-rebench" +_venv_lock = threading.Lock() + + +def rebench_python() -> str: + """The rebench grader's interpreter, creating its venv on first use.""" + python = REBENCH_VENV / "bin" / "python" + with _venv_lock: + if not python.exists(): + subprocess.run(["uv", "venv", "--quiet", "--python", "3.12", str(REBENCH_VENV)], check=True) + subprocess.run(["uv", "pip", "install", "--quiet", "--python", str(python), + REBENCH_HARNESS], check=True) + return str(python) + + +def grader_command(grader: str) -> list[str]: + if grader == "swebench": + return [sys.executable, "-m", "swebench.harness.run_evaluation"] + if grader == "rebench": + # Images come from each row's image_name; keep them for --remove-images. + return [rebench_python(), "-m", "swebench.harness.run_evaluation", + "--cache_level", "instance", "--namespace", "swerebench"] + raise ValueError(f"unknown grader {grader!r}") + + +def report_path(run_dir: Path, run_id: str, model_name: str, instance_id: str) -> Path: + return (run_dir / "logs" / "run_evaluation" / run_id / model_name.replace("/", "__") + / instance_id / "report.json") + + +def evaluate_instance(*, run_dir: Path, run_id: str, model_name: str, instance_id: str, + patch: str, timeout_s: int = 1_800, grader: str = "swebench") -> dict: + """Grade one prediction; returns ``{resolved, status, ...}`` for the report.""" + if not patch.strip(): + return {"resolved": False, "status": "empty_patch"} + instance_dir = run_dir / "instances" / instance_id + log_path = instance_dir / "eval.log" + # Both harnesses skip an instance that already has a report for this run + # id, so a re-run task would silently inherit its previous verdict. + stale = report_path(run_dir, run_id, model_name, instance_id).parent + if stale.exists(): + shutil.rmtree(stale) + command = [ + *grader_command(grader), + "--dataset_name", str(run_dir / "dataset.json"), + "--split", "test", + "--instance_ids", instance_id, + "--predictions_path", str(run_dir / "predictions.jsonl"), + "--max_workers", "1", + "--run_id", run_id, + "--timeout", str(timeout_s), + "--report_dir", str(instance_dir / "eval-report"), + ] + with log_path.open("w") as log: + proc = subprocess.run(command, cwd=run_dir, env=docker_env(), stdout=log, + stderr=subprocess.STDOUT, timeout=timeout_s + 900) + report_file = report_path(run_dir, run_id, model_name, instance_id) + if not report_file.exists(): + return {"resolved": False, "status": "eval_error", "exit_code": proc.returncode} + report = json.loads(report_file.read_text()).get(instance_id, {}) + return { + "resolved": bool(report.get("resolved")), + "status": "resolved" if report.get("resolved") else ( + "unresolved" if report.get("patch_successfully_applied") else "patch_failed"), + "patch_applied": bool(report.get("patch_successfully_applied")), + "tests_status": report.get("tests_status"), + } diff --git a/benchmarks/swebench/code4me_swebench/prompt.py b/benchmarks/swebench/code4me_swebench/prompt.py new file mode 100644 index 0000000..095d155 --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/prompt.py @@ -0,0 +1,33 @@ +"""The task message the agent receives (the only user turn of a run).""" + +from __future__ import annotations + +TEMPLATE = """\ +You are working on your own on a software engineering task. The repository is \ +checked out at {workspace}, which is your workspace root; use paths relative to \ +it. Its development environment is already installed: `python` and the \ +project's test tools are on PATH. + + +{problem_statement} + + +Change the repository's non-test source files so that the issue above is resolved. + +- Tests that check the fix already exist and are hidden from you. Do not modify \ +existing tests or test configuration. +- First find the relevant code and reproduce the problem, for example with a \ +small script. Then make the smallest change that fixes it and confirm the \ +reproduction now behaves correctly. Run the most relevant existing tests to \ +check that nothing else broke. +- Think about edge cases the issue mentions or implies. +- Nobody will answer questions during this task. Do not ask for confirmation or \ +clarification: make reasonable assumptions and keep going until the fix is done. +- Do not commit, stash or switch branches; leave your changes in the working \ +tree. Remove scratch files you created once you are done. +- End with a short summary of what you changed. +""" + + +def task_prompt(problem_statement: str, *, workspace: str) -> str: + return TEMPLATE.format(workspace=workspace, problem_statement=problem_statement.strip()) diff --git a/benchmarks/swebench/code4me_swebench/report.py b/benchmarks/swebench/code4me_swebench/report.py new file mode 100644 index 0000000..ef38b6f --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/report.py @@ -0,0 +1,243 @@ +"""Run-level report: resolve rate with a confidence interval, plus telemetry totals.""" + +from __future__ import annotations + +import csv +import json +import math +from collections import Counter +from pathlib import Path +from statistics import mean, median, stdev + +REFERENCES = Path(__file__).resolve().parents[1] / "references.json" + +COLUMNS = ( + "instance_id", "resolved", "eval_status", "agent_status", "run_status", "stop_reason", + "model_calls", "prompt_tokens", "cached_prompt_tokens", "completion_tokens", + "tool_calls", "tool_failures", "test_runs", "files_edited", "patch_lines", + "agent_wall_s", "cost_usd", +) + + +def wilson_interval(successes: int, total: int, z: float = 1.96) -> tuple[float, float]: + if total == 0: + return (0.0, 0.0) + p = successes / total + denominator = 1 + z * z / total + centre = (p + z * z / (2 * total)) / denominator + half = z * math.sqrt(p * (1 - p) / total + z * z / (4 * total * total)) / denominator + return (max(0.0, centre - half), min(1.0, centre + half)) + + +def _load(path: Path) -> dict: + return json.loads(path.read_text()) if path.exists() else {} + + +def instance_rows(run_dir: Path) -> list[dict]: + manifest = _load(run_dir / "manifest.json") + rows = [] + for instance_id in manifest.get("instance_ids", []): + directory = run_dir / "instances" / instance_id + result, evaluation = _load(directory / "result.json"), _load(directory / "eval.json") + tel, host = _load(directory / "telemetry.json"), _load(directory / "host.json") + patch = (directory / "patch.diff").read_text() if (directory / "patch.diff").exists() else "" + if (directory / "infra_error.json").exists(): + agent_status = "infra_error" + elif not (directory / "prediction.json").exists(): + agent_status = "not_run" + elif host.get("agent_timed_out"): + agent_status = "timeout" + else: + agent_status = result.get("status", "missing") + rows.append({ + "instance_id": instance_id, + "resolved": evaluation.get("resolved"), + "eval_status": evaluation.get("status"), + "agent_status": agent_status, + "run_status": result.get("run_status"), + "stop_reason": result.get("stop_reason"), + "model_calls": tel.get("model_calls", 0), + "prompt_tokens": tel.get("prompt_tokens", 0), + "cached_prompt_tokens": tel.get("cached_prompt_tokens", 0), + "completion_tokens": tel.get("completion_tokens", 0), + "tool_calls": sum((tel.get("tool_calls") or {}).values()), + "tool_failures": tel.get("tool_failures", 0), + "test_runs": tel.get("test_runs", 0), + "files_edited": len(tel.get("files_edited") or []), + "patch_lines": sum(1 for line in patch.splitlines() + if line[:1] in "+-" and line[:3] not in ("+++", "---")), + "agent_wall_s": host.get("agent_wall_s"), + "cost_usd": tel.get("cost_usd"), + }) + return rows + + +def build_report(run_dir: Path) -> dict: + manifest = _load(run_dir / "manifest.json") + rows = instance_rows(run_dir) + graded = [row for row in rows if row["resolved"] is not None] + resolved = sum(1 for row in graded if row["resolved"]) + total = len(rows) + low, high = wilson_interval(resolved, total) + + def counts(key: str) -> dict: + out: dict = {} + for row in rows: + out[str(row[key])] = out.get(str(row[key]), 0) + 1 + return dict(sorted(out.items())) + + def stat(key: str) -> dict: + values = [row[key] for row in rows if isinstance(row[key], (int, float))] + return {"total": round(sum(values), 3), "median": median(values) if values else None} + + report = { + "run": run_dir.name, + "model": manifest.get("settings", {}).get("model"), + "subset": manifest.get("subset"), + "instances": total, + "graded": len(graded), + # Ungraded tasks count as unresolved: the rate is over the whole subset. + "resolved": resolved, + "resolve_rate": round(resolved / total, 4) if total else None, + "resolve_rate_95ci": [round(low, 4), round(high, 4)], + "agent_status": counts("agent_status"), + "run_status": counts("run_status"), + "eval_status": counts("eval_status"), + "model_calls": stat("model_calls"), + "prompt_tokens": stat("prompt_tokens"), + "cached_prompt_tokens": stat("cached_prompt_tokens"), + "completion_tokens": stat("completion_tokens"), + "tool_calls": stat("tool_calls"), + "agent_wall_s": stat("agent_wall_s"), + # Provider-reported spend; total 0 when the provider reports none. + "cost_usd": stat("cost_usd"), + "runtime": manifest.get("runtime"), + "settings": manifest.get("settings"), + } + return report + + +def write_report(run_dir: Path) -> dict: + report = build_report(run_dir) + (run_dir / "report.json").write_text(json.dumps(report, indent=2) + "\n") + with (run_dir / "instances.csv").open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=COLUMNS) + writer.writeheader() + writer.writerows(instance_rows(run_dir)) + return report + + +def mcnemar_exact_p(only_a: int, only_b: int) -> float: + """Two-sided exact McNemar test on the discordant pairs.""" + n = only_a + only_b + if n == 0: + return 1.0 + tail = sum(math.comb(n, k) for k in range(min(only_a, only_b) + 1)) / 2 ** n + return min(1.0, 2 * tail) + + +def compare_runs(run_a: Path, run_b: Path) -> dict: + """Paired comparison of two runs over the tasks both contain.""" + rows_a = {row["instance_id"]: row for row in instance_rows(run_a)} + rows_b = {row["instance_id"]: row for row in instance_rows(run_b)} + shared = sorted(set(rows_a) & set(rows_b)) + cells = {"both": 0, "only_a": 0, "only_b": 0, "neither": 0} + for instance_id in shared: + a, b = bool(rows_a[instance_id]["resolved"]), bool(rows_b[instance_id]["resolved"]) + cells["both" if a and b else "only_a" if a else "only_b" if b else "neither"] += 1 + + def total(rows: dict, key: str) -> int: + return sum(int(rows[i][key] or 0) for i in shared) + + return { + "a": run_a.name, "b": run_b.name, "tasks": len(shared), + "resolved_a": cells["both"] + cells["only_a"], + "resolved_b": cells["both"] + cells["only_b"], + **cells, + "mcnemar_p": round(mcnemar_exact_p(cells["only_a"], cells["only_b"]), 4), + "prompt_tokens_a": total(rows_a, "prompt_tokens"), + "prompt_tokens_b": total(rows_b, "prompt_tokens"), + "model_calls_a": total(rows_a, "model_calls"), + "model_calls_b": total(rows_b, "model_calls"), + } + + +def _window_stats(per_run: list[list[dict]], task_ids: list[str]) -> dict: + """The SWE-rebench leaderboard's metrics over repeated runs of the same tasks: + resolved rate = mean of the per-run rates (every task in the denominator; an + unrun or ungraded task is unresolved); SEM = sample standard deviation of + those rates / sqrt(runs); pass@n = tasks resolved in at least one run; + tokens/problem = mean over task-runs of input (cached included) + output; + cached share = mean over task-runs of cached / input.""" + wanted = set(task_ids) + rates, solved_any = [], set() + task_runs = [] + for rows in per_run: + mine = [row for row in rows if row["instance_id"] in wanted] + task_runs.extend(mine) + resolved = {row["instance_id"] for row in mine if row["resolved"]} + solved_any |= resolved + rates.append(100.0 * len(resolved) / len(wanted)) + runs = len(rates) + used = [row for row in task_runs if (row["prompt_tokens"] or 0) > 0] + costs = [row["cost_usd"] for row in task_runs if isinstance(row.get("cost_usd"), (int, float))] + return { + "tasks": len(wanted), + "runs": runs, + "resolved_rate": round(mean(rates), 2), + "sem": round(stdev(rates) / math.sqrt(runs), 2) if runs > 1 else None, + "per_run_rates": [round(rate, 2) for rate in rates], + f"pass_at_{runs}": round(100.0 * len(solved_any) / len(wanted), 2), + "tokens_per_problem": round(mean((row["prompt_tokens"] or 0) + (row["completion_tokens"] or 0) + for row in task_runs)) if task_runs else None, + "cached_share": round(100.0 * mean((row["cached_prompt_tokens"] or 0) / row["prompt_tokens"] + for row in used), 2) if used else None, + # Billed by the provider (OpenRouter's usage.cost); None when it reports none. + "cost_per_problem": round(mean(costs), 4) if costs and len(costs) == len(task_runs) else None, + "model_calls_per_problem": round(mean(row["model_calls"] or 0 for row in task_runs), 1) if task_runs else None, + "agent_status": dict(sorted(Counter(row["agent_status"] for row in task_runs).items())), + } + + +def aggregate_runs(run_dirs: list[Path], *, windows: list[tuple[str, str]] = (), + reference: str | None = None) -> dict: + """Leaderboard-style results for repeated runs of one task set, overall and per + time window [start, end) by task creation date, optionally next to a published + reference (``references.json``).""" + manifests = [_load(run_dir / "manifest.json") for run_dir in run_dirs] + ids = manifests[0].get("instance_ids", []) + for run_dir, manifest in zip(run_dirs, manifests): + if manifest.get("instance_ids") != ids or manifest.get("settings") != manifests[0].get("settings"): + raise SystemExit(f"{run_dir.name} has other tasks or settings than {run_dirs[0].name}") + per_run = [instance_rows(run_dir) for run_dir in run_dirs] + created = {row["instance_id"]: str(row.get("created_at"))[:10] + for row in json.loads((run_dirs[0] / "dataset.json").read_text())} + span = (min(created[i] for i in ids), max(created[i] for i in ids)) if ids else (None, None) + result = { + "runs": [run_dir.name for run_dir in run_dirs], + "dataset": manifests[0].get("dataset"), "split": manifests[0].get("split"), + "dataset_revision": manifests[0].get("dataset_revision"), + "model": manifests[0].get("settings", {}).get("model"), + "created": span, + "overall": _window_stats(per_run, ids), + "windows": {}, + } + for start, end in windows: + window_ids = [i for i in ids if start <= created[i] < end] + if window_ids: + result["windows"][f"{start}..{end}"] = _window_stats(per_run, window_ids) + if reference: + refs = json.loads(REFERENCES.read_text())["references"] + if reference not in refs: + raise SystemExit(f"unknown reference {reference!r}; known: {', '.join(sorted(refs))}") + result["reference"] = {"name": reference, **{k: v for k, v in refs[reference].items() if k != "windows"}} + for key, ref in refs[reference]["windows"].items(): + ours = result["windows"].get(key) + # Only the same task set is comparable (an incomplete window is not). + if not ours or ours["tasks"] != ref["tasks"]: + continue + diff = ours["resolved_rate"] - ref["resolved_rate"] + spread = math.sqrt((ours["sem"] or 0.0) ** 2 + ref["sem"] ** 2) + ours["reference"] = {**ref, "difference": round(diff, 2), + "z": round(diff / spread, 2) if spread else None} + return result diff --git a/benchmarks/swebench/code4me_swebench/runner.py b/benchmarks/swebench/code4me_swebench/runner.py new file mode 100644 index 0000000..3d42251 --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/runner.py @@ -0,0 +1,421 @@ +"""Runs the agent on SWE-bench tasks, one task container per instance. + +Run directory layout:: + + /manifest.json settings, agent build, dataset, instance list + /dataset.json the selected dataset rows (graded from this file) + /predictions.jsonl SWE-bench predictions, rebuilt after every task + /instances// trace.jsonl (agent telemetry), result.json, + agent-config.json, patch.diff, agent.log, + telemetry.json, eval.json, eval.log +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import shutil +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, replace +from datetime import datetime, timezone +from pathlib import Path +from time import perf_counter +from typing import TYPE_CHECKING + +from code4me_swebench import docker_cli, evaluate, sandbox, telemetry +from code4me_swebench.prompt import task_prompt +from code4me_swebench.settings import ( + OUT_DIR, + RUNTIME_MOUNT, + TASK_DIR, + WORKSPACE, + RunSettings, + agent_config, +) + +if TYPE_CHECKING: + from code4me_swebench.runtime import Runtime + +logger = logging.getLogger(__name__) +DRIVER = Path(__file__).resolve().parent / "driver.py" +_predictions_lock = threading.Lock() +MAX_CONSECUTIVE_OUTAGES = 3 +# Model-service failures that say nothing about the agent (rate limit, overload, +# outage), and the service refusing to serve at all: a bad or exhausted key +# (401/402/403, e.g. an OpenRouter key at its credit cap) or no model or pinned +# upstream to serve it (404). Anything else, such as 400 for a malformed request, +# is the agent's. +RETRYABLE_STATUSES = frozenset({408, 409, 429, 500, 502, 503, 504}) +REFUSED_STATUSES = frozenset({401, 402, 403, 404}) + + +@dataclass(frozen=True) +class RunOptions: + evaluate: bool = True + remove_images: bool = False + force: bool = False + memory: str | None = None + cpus: str | None = None + eval_timeout_s: int = 1_800 + # "swebench" (swebench 5.0.2) or "rebench" (the SWE-rebench harness fork). + grader: str = "swebench" + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _write_json(path: Path, data: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") + tmp.replace(path) + + +def prepare_run(run_dir: Path, *, rows: list[dict], settings: RunSettings, + runtime: Runtime, subset: str, dataset_name: str, split: str, + grader: str, dataset_revision: str | None = None) -> None: + run_dir.mkdir(parents=True, exist_ok=True) + manifest_path = run_dir / "manifest.json" + ids = [row["instance_id"] for row in rows] + if manifest_path.exists(): + previous = json.loads(manifest_path.read_text()) + if previous["settings"] != settings.as_dict() or previous["instance_ids"] != ids: + raise SystemExit( + f"{run_dir} was started with other settings or instances; use a new --run-name" + ) + if previous["runtime"]["wheel_sha256"] != runtime.wheel_sha256: + logger.warning("Agent code changed since this run started (wheel %s -> %s)", + previous["runtime"]["wheel_sha256"][:12], runtime.wheel_sha256[:12]) + return + _write_json(run_dir / "dataset.json", rows) + _write_json(manifest_path, { + "created_at": _now(), + "subset": subset, + "dataset": dataset_name, + "split": split, + **({"dataset_revision": dataset_revision} if dataset_revision else {}), + "grader": grader, + "instance_ids": ids, + "settings": settings.as_dict(), + "runtime": runtime.as_dict(), + }) + + +def write_predictions(run_dir: Path) -> None: + with _predictions_lock: + lines = [] + for path in sorted((run_dir / "instances").glob("*/prediction.json")): + lines.append(json.dumps(json.loads(path.read_text()), sort_keys=True)) + tmp = run_dir / "predictions.jsonl.tmp" + tmp.write_text("".join(line + "\n" for line in lines)) + tmp.replace(run_dir / "predictions.jsonl") + + +def _container_name(run_name: str, instance_id: str) -> str: + return re.sub(r"[^a-zA-Z0-9_.-]", "-", f"c4m-swe-{run_name}-{instance_id}")[:120] + + +# SWE-bench images keep conda in /opt/miniconda3, SWE-rebench images in /opt/conda. +_ACTIVATE_TESTBED = ( + 'for a in /opt/miniconda3/bin/activate /opt/conda/bin/activate; do ' + '[ -f "$a" ] && source "$a" testbed >/dev/null 2>&1 && break; done; ' + 'printf "%s\\n%s\\n" "$PATH" "${CONDA_PREFIX:-}"' +) +# Files some SWE-rebench images ship in / that describe the task (and possibly +# its fix). The agent's shell could read them, so they go before it starts. +LEAK_FILES = ("/issue.md", "/swebench_instance.json") + + +def _activated_env(container: str) -> list[str]: + """PATH/CONDA_PREFIX of the image's ``testbed`` conda env (as its eval script uses).""" + probe = docker_cli.exec_in(container, "bash", "-c", _ACTIVATE_TESTBED) + lines = probe.stdout.splitlines() + if probe.returncode == 0 and len(lines) >= 2 and lines[1].endswith("/envs/testbed"): + return [f"PATH={lines[0]}", f"CONDA_PREFIX={lines[1]}"] + raise RuntimeError(f"{container}: no testbed conda env found") + + +def _remove_leak_files(container: str) -> list[str]: + present = docker_cli.exec_in(container, "ls", "-1", *LEAK_FILES).stdout.split() + if present: + docker_cli.exec_in(container, "rm", "-f", *present, check=True) + return present + + +def _git(container: str, *args: str) -> str: + return docker_cli.exec_in(container, "git", "-C", WORKSPACE, *args, check=True).stdout + + +def _extract_patch(container: str, base: str) -> str: + """Everything the agent changed, staged, as a binary-safe diff against ``base``.""" + docker_cli.exec_in(container, "git", "-C", WORKSPACE, "add", "-A", check=True) + return docker_cli.exec_in( + container, "git", "-C", WORKSPACE, "-c", "core.fileMode=false", + "diff", "--cached", "--binary", "--no-color", base, check=True, + ).stdout + + +def run_instance(row: dict, *, run_dir: Path, run_name: str, settings: RunSettings, + runtime: Runtime, options: RunOptions) -> dict: + instance_id = row["instance_id"] + instance_dir = run_dir / "instances" / instance_id + done_marker = instance_dir / "prediction.json" + if done_marker.exists() and not options.force: + return _instance_record(instance_dir, instance_id, skipped=True) + instance_dir.mkdir(parents=True, exist_ok=True) + for stale in ("trace.jsonl", "result.json", "eval.json", "telemetry.json"): + (instance_dir / stale).unlink(missing_ok=True) + + image = row["image"] + container = _container_name(run_name, instance_id) + session_id = f"swebench-{run_name}-{instance_id}" + pull_started = perf_counter() + docker_cli.pull(image, platform=settings.platform) + pull_s = round(perf_counter() - pull_started, 3) + started = perf_counter() + docker_cli.start_container( + name=container, image=image, platform=settings.platform, + mounts=[f"{runtime.volume}:{RUNTIME_MOUNT}:ro"], + memory=options.memory, cpus=options.cpus, network=sandbox.NETWORK, + ) + host: dict = {"instance_id": instance_id, "image": image, "started_at": _now(), + "image_pull_s": pull_s, "network": settings.network} + container_ip = docker_cli.container_ip(container, sandbox.NETWORK) + try: + env = _activated_env(container) + host["removed_leak_files"] = _remove_leak_files(container) + base = _git(container, "rev-parse", "HEAD").strip() + host["base_commit"] = base + # The images commit their setup ("SWE-bench") on top of the dataset's + # base commit; the patch is taken against the image HEAD either way. + parent = docker_cli.exec_in(container, "git", "-C", WORKSPACE, "rev-parse", "HEAD^") + host["head_on_dataset_base"] = row.get("base_commit") in (base, parent.stdout.strip()) + host["preexisting_changes"] = _git(container, "status", "--porcelain").strip() + task = { + "instance_id": instance_id, + "run_id": f"{run_name}-{instance_id}", + "out_dir": OUT_DIR, + "prompt": task_prompt(row["problem_statement"], workspace=WORKSPACE), + "agent_config": agent_config(settings, session_id=session_id), + "allowed_tools": list(settings.tools), + } + _write_json(instance_dir / "task.json", task) + docker_cli.exec_in(container, "mkdir", "-p", OUT_DIR, check=True) + docker_cli.copy_to(container, DRIVER, f"{TASK_DIR}/driver.py") + docker_cli.copy_to(container, instance_dir / "task.json", f"{TASK_DIR}/task.json") + agent = docker_cli.exec_in( + container, f"{RUNTIME_MOUNT}/venv/bin/python", f"{TASK_DIR}/driver.py", + f"{TASK_DIR}/task.json", + workdir=WORKSPACE, + # `-e NAME` without a value forwards the host value: the key never + # appears on a command line or on disk. + env=[*env, *sandbox.proxy_env(), settings.api_key_env, "CODE4ME_AGENT_LOG_LEVEL=INFO"], + timeout=settings.task_timeout_s, + ) + host["agent_exit_code"] = agent.returncode + host["agent_timed_out"] = agent.timed_out + if agent.timed_out: + docker_cli.exec_in(container, "pkill", "-f", f"{TASK_DIR}/driver.py") + (instance_dir / "agent.log").write_text(agent.stdout + agent.stderr) + patch = _extract_patch(container, base) + docker_cli.copy_from(container, f"{OUT_DIR}/.", instance_dir) + # Audit: connections this container tried that the proxy refused. + host["egress_denied"] = [line for line in sandbox.denied_attempts(since=host["started_at"]) + if container_ip and f" {container_ip} " in line] + finally: + docker_cli.remove_container(container) + host["agent_wall_s"] = round(perf_counter() - started, 3) + (instance_dir / "patch.diff").write_text(patch) + _write_json(instance_dir / "host.json", host) + _write_json(instance_dir / "telemetry.json", telemetry.summarize(instance_dir / "trace.jsonl")) + outage = provider_outage(instance_dir) + if outage: + # The model service failed (rate limit, outage), not the agent: no + # prediction is written, so a resumed run retries the task. + _write_json(instance_dir / "infra_error.json", outage) + if options.remove_images: + docker_cli.remove_image(image) + return {"instance_id": instance_id, "agent_status": "infra_error", **outage} + (instance_dir / "infra_error.json").unlink(missing_ok=True) + _write_json(done_marker, { + "instance_id": instance_id, + "model_name_or_path": settings.model_name_or_path, + "model_patch": patch, + }) + write_predictions(run_dir) + if options.evaluate: + result = evaluate.evaluate_instance( + run_dir=run_dir, run_id=run_name, model_name=settings.model_name_or_path, + instance_id=instance_id, patch=patch, timeout_s=options.eval_timeout_s, + grader=options.grader, + ) + _write_json(instance_dir / "eval.json", result) + if options.remove_images: + docker_cli.remove_image(image) + return _instance_record(instance_dir, instance_id) + + +def provider_outage(instance_dir: Path) -> dict | None: + """Details when the agent stopped because its model calls failed, else None. + + The agent ends a turn with stop_reason "error" only when the provider call + itself failed after retries (adapters: provider_request_failed). + """ + result_path = instance_dir / "result.json" + result = json.loads(result_path.read_text()) if result_path.exists() else {} + if not (result.get("run_status") == "failed" and result.get("stop_reason") == "error"): + return None + # The agent reports a failed main-loop model call on agent.adapter.loop_failed + # (error_type in the payload, status_code/attempts as metrics). + failures = [event for event in telemetry.read_events(instance_dir / "trace.jsonl") + if event.get("event_type") == "agent.adapter.loop_failed" + and (event.get("payload") or {}).get("failure_reason") == "provider_request_failed"] + last = failures[-1] if failures else {} + status = (last.get("metrics") or {}).get("status_code") + if status is not None and status not in RETRYABLE_STATUSES | REFUSED_STATUSES: + # e.g. 400: the provider rejected what the agent sent. That is the + # agent's failure and is graded like any other result. + return None + return { + "reason": "provider_request_failed", + "status_code": status, + "error_type": (last.get("payload") or {}).get("error_type"), + "message": (result.get("final_response") or "")[:300], + } + + +def _instance_record(instance_dir: Path, instance_id: str, *, skipped: bool = False) -> dict: + def load(name: str) -> dict: + path = instance_dir / name + return json.loads(path.read_text()) if path.exists() else {} + + result, evaluation = load("result.json"), load("eval.json") + return { + "instance_id": instance_id, + "skipped": skipped, + "agent_status": result.get("status", "missing"), + "resolved": evaluation.get("resolved"), + "eval_status": evaluation.get("status"), + } + + +def run_all(rows: list[dict], *, run_dir: Path, run_name: str, settings: RunSettings, + runtime: Runtime, options: RunOptions, workers: int) -> list[dict]: + return run_repeated(rows, runs=[(run_dir, run_name)], settings=settings, runtime=runtime, + options=options, workers=workers) + + +def run_repeated(rows: list[dict], *, runs: list[tuple[Path, str]], settings: RunSettings, + runtime: Runtime, options: RunOptions, workers: int) -> list[dict]: + """Run every task once per run directory (independent repeats, as the SWE-rebench + leaderboard's 5 runs), task by task: a task's image is pulled once for all of + its attempts and removed after the last one.""" + if not os.environ.get(settings.api_key_env): + raise SystemExit(f"set {settings.api_key_env} to the provider API key first") + if settings.network != "model-api-only": + raise SystemExit(f"unsupported network policy {settings.network!r}") + allowed = sandbox.ensure(settings.base_url) + logger.info("Sandbox: task containers on %s, egress only to %s:443", sandbox.NETWORK, allowed) + attempt_options = replace(options, remove_images=False) + lock = threading.Lock() + stop = threading.Event() + state = {"consecutive_outages": 0, "done": 0} + total = len(rows) * len(runs) + records: list[dict] = [] + + def attempt(row: dict, run_dir: Path, run_name: str) -> dict: + try: + return run_instance(row, run_dir=run_dir, run_name=run_name, + settings=settings, runtime=runtime, options=attempt_options) + except Exception as exc: # noqa: BLE001 - one broken task must not stop the run + logger.exception("%s failed", row["instance_id"]) + return {"instance_id": row["instance_id"], "agent_status": "harness_error", + "error": f"{type(exc).__name__}: {exc}"} + + def one(row: dict) -> None: + try: + for run_dir, run_name in runs: + if stop.is_set(): + return + record = {**attempt(row, run_dir, run_name), "run": run_name} + with lock: + records.append(record) + state["done"] += 1 + logger.info("[%d/%d] %s %s agent=%s eval=%s", state["done"], total, run_name, + record["instance_id"], record.get("agent_status"), record.get("eval_status")) + outage = record.get("agent_status") == "infra_error" + state["consecutive_outages"] = state["consecutive_outages"] + 1 if outage else 0 + if state["consecutive_outages"] >= MAX_CONSECUTIVE_OUTAGES and not stop.is_set(): + # A rate limit, outage or spent key would fail every remaining attempt. + stop.set() + logger.error("Stopping: %d attempts in a row failed on the model service (%s). " + "Re-run the same command later; finished attempts are kept.", + state["consecutive_outages"], + record.get("status_code") or record.get("error_type")) + finally: + if options.remove_images: + docker_cli.remove_image(row["image"]) + + with ThreadPoolExecutor(max_workers=max(1, workers)) as pool: + futures = [pool.submit(one, row) for row in rows] + for future in as_completed(futures): + future.result() + return records + + +ATTEMPT_FILES = ( + "trace.jsonl", "result.json", "agent.log", "host.json", "telemetry.json", "task.json", + "agent-config.json", "patch.diff", "prediction.json", "eval.json", "eval.log", "infra_error.json", +) + + +def requeue_outages(run_dir: Path) -> list[str]: + """Release tasks whose finished attempt was a model-service outage. + + The attempt's files move to ``attempts//`` (kept as evidence) and its + grader report is removed, so the next ``run``/``baseline-mini`` redoes it. + """ + from code4me_swebench.baseline_mini import PROVIDER_FAILURES + + manifest = json.loads((run_dir / "manifest.json").read_text()) + model_dirs = list((run_dir / "logs" / "run_evaluation" / run_dir.name).glob("*")) + released = [] + for instance_id in manifest["instance_ids"]: + directory = run_dir / "instances" / instance_id + if not directory.exists(): + continue + if "baseline" in manifest: + tel = directory / "telemetry.json" + exit_status = json.loads(tel.read_text()).get("exit_status") if tel.exists() else None + outage = exit_status in PROVIDER_FAILURES or (directory / "infra_error.json").exists() + else: + if (directory / "infra_error.json").exists() and not provider_outage(directory): + # Labelled an outage by an older rule (e.g. HTTP 400): it is the + # agent's own failure, so keep the attempt and let `evaluate` grade it. + (directory / "infra_error.json").unlink() + (directory / "prediction.json").write_text(json.dumps({ + "instance_id": instance_id, + "model_name_or_path": f"code4me2-agent__{manifest['settings']['model']}", + "model_patch": (directory / "patch.diff").read_text(), + }, sort_keys=True)) + continue + outage = bool(provider_outage(directory)) or (directory / "infra_error.json").exists() + if not outage: + continue + attempts = directory / "attempts" + target = attempts / str(len(list(attempts.glob("*"))) + 1 if attempts.exists() else 1) + target.mkdir(parents=True) + for name in ATTEMPT_FILES: + if (directory / name).exists(): + (directory / name).rename(target / name) + for model_dir in model_dirs: + stale = model_dir / instance_id + if stale.exists(): + shutil.rmtree(stale) + released.append(instance_id) + write_predictions(run_dir) + return released diff --git a/benchmarks/swebench/code4me_swebench/runtime.py b/benchmarks/swebench/code4me_swebench/runtime.py new file mode 100644 index 0000000..7f45704 --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/runtime.py @@ -0,0 +1,130 @@ +"""Builds the agent runtime that is mounted read-only into every task container. + +The packaged native runtime (PyInstaller, built on Ubuntu 24.04) needs a newer +glibc than the SWE-bench images (Ubuntu 22.04), so the agent runs from its +wheel on a portable python-build-standalone interpreter instead, with the same +locked dependencies as the release (``packaging/requirements-runtime.lock``). + +The volume name is derived from the wheel, the lock and the platform, so an +agent change produces a new runtime instead of silently reusing a stale one. +""" + +from __future__ import annotations + +import hashlib +import shutil +import subprocess +import tempfile +import zipfile +from dataclasses import dataclass +from pathlib import Path + +from code4me_swebench import docker_cli +from code4me_swebench.settings import RUNTIME_MOUNT + +SERVER_ROOT = Path(__file__).resolve().parents[3] +LOCK_FILE = SERVER_ROOT / "packaging" / "requirements-runtime.lock" +BUILDER_IMAGE = "ghcr.io/astral-sh/uv:0.9-bookworm-slim" +PYTHON_VERSION = "3.13" + + +@dataclass(frozen=True) +class Runtime: + volume: str + wheel_name: str + # Content hash (member names and bytes), stable across rebuilds. + wheel_sha256: str + lock_sha256: str + git_commit: str + git_dirty: bool + platform: str + + def as_dict(self) -> dict[str, object]: + return dict(self.__dict__) + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _wheel_content_sha256(wheel: Path) -> str: + """Hash of the wheel's member names and bytes, ignoring zip timestamps.""" + digest = hashlib.sha256() + with zipfile.ZipFile(wheel) as archive: + for name in sorted(archive.namelist()): + digest.update(name.encode() + b"\0" + archive.read(name) + b"\0") + return digest.hexdigest() + + +def _git(*args: str) -> str: + return subprocess.run( + ["git", "-C", str(SERVER_ROOT), *args], capture_output=True, text=True, check=True + ).stdout.strip() + + +def build_wheel(out_dir: Path) -> Path: + out_dir.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["uv", "build", "--wheel", "--quiet", "--out-dir", str(out_dir), str(SERVER_ROOT)], + check=True, + ) + wheels = sorted(out_dir.glob("code4me2_agent-*.whl")) + if len(wheels) != 1: + raise RuntimeError(f"expected one agent wheel in {out_dir}, found {wheels}") + return wheels[0] + + +def ensure_runtime(*, platform: str, rebuild: bool = False) -> Runtime: + """Build (or reuse) the runtime volume for the agent in this checkout.""" + with tempfile.TemporaryDirectory(prefix="code4me-swebench-") as scratch: + scratch_dir = Path(scratch) + wheel = build_wheel(scratch_dir / "wheels") + wheel_sha = _wheel_content_sha256(wheel) + lock_sha = _sha256(LOCK_FILE) + key = hashlib.sha256(f"{wheel_sha}:{lock_sha}:{platform}:{PYTHON_VERSION}".encode()) + arch = platform.split("/")[-1] + volume = f"code4me-swebench-runtime-{arch}-{key.hexdigest()[:12]}" + runtime = Runtime( + volume=volume, + wheel_name=wheel.name, + wheel_sha256=wheel_sha, + lock_sha256=lock_sha, + git_commit=_git("rev-parse", "HEAD"), + git_dirty=bool(_git("status", "--porcelain", "--", "src/code4me2_agent", "pyproject.toml")), + platform=platform, + ) + if docker_cli.volume_exists(volume) and not rebuild: + return runtime + docker_cli.docker("volume", "rm", "-f", volume, check=False) + docker_cli.docker("volume", "create", volume) + # Inputs travel by `docker cp`, not a bind mount: Colima shares only + # $HOME with its VM, so a macOS temp directory would mount empty. + inputs = scratch_dir / "inputs" + inputs.mkdir() + shutil.copy2(wheel, inputs / wheel.name) + shutil.copy2(LOCK_FILE, inputs / "requirements-runtime.lock") + venv_python = f"{RUNTIME_MOUNT}/venv/bin/python" + script = " && ".join([ + f"UV_PYTHON_INSTALL_DIR={RUNTIME_MOUNT}/python uv python install {PYTHON_VERSION}", + f"UV_PYTHON_INSTALL_DIR={RUNTIME_MOUNT}/python uv venv {RUNTIME_MOUNT}/venv " + f"--python {PYTHON_VERSION} --python-preference only-managed", + f"uv pip install --python {venv_python} --compile-bytecode " + "-r /inputs/requirements-runtime.lock", + f"uv pip install --python {venv_python} --compile-bytecode --no-deps /inputs/{wheel.name}", + f"{venv_python} -c 'import code4me2_agent.echo, acp, openai, mcp'", + ]) + builder = f"{volume}-builder" + docker_cli.remove_container(builder) + try: + docker_cli.docker( + "create", "--name", builder, "--platform", platform, + "-v", f"{volume}:{RUNTIME_MOUNT}", BUILDER_IMAGE, "sh", "-c", script, + ) + docker_cli.copy_to(builder, inputs, "/inputs") + docker_cli.docker("start", "-a", builder, timeout=1_800) + except BaseException: + docker_cli.docker("volume", "rm", "-f", volume, check=False) + raise + finally: + docker_cli.remove_container(builder) + return runtime diff --git a/benchmarks/swebench/code4me_swebench/sandbox.py b/benchmarks/swebench/code4me_swebench/sandbox.py new file mode 100644 index 0000000..14df239 --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/sandbox.py @@ -0,0 +1,74 @@ +"""Network sandbox for task containers: the model API host and nothing else. + +Our agent runs inside the task container, so it needs one way out (its model +API). The container joins an ``--internal`` Docker network, which has no route +to the outside, and gets HTTP(S)_PROXY pointing at the egress proxy: the only +member of both that network and the default bridge. The proxy tunnels to the +allowlisted host only, so pip, git and GitHub are unreachable, and every attempt +is logged (``docker logs code4me-swebench-proxy``). +""" + +from __future__ import annotations + +from pathlib import Path +from urllib.parse import urlparse + +from code4me_swebench import docker_cli + +NETWORK = "code4me-swebench-internal" +PROXY = "code4me-swebench-proxy" +PROXY_IMAGE = "python:3.12-alpine" +PROXY_PORT = 3128 +PROXY_SOURCE = Path(__file__).resolve().parent / "egress_proxy.py" + + +def model_host(base_url: str) -> str: + host = (urlparse(base_url).hostname or "").lower() + if not host: + raise ValueError(f"cannot sandbox: no host in base URL {base_url!r}") + return host + + +def proxy_env() -> list[str]: + """Exec environment for processes in a task container (also inherited by + the agent's commands, whose env allowlist passes *_PROXY).""" + url = f"http://{PROXY}:{PROXY_PORT}" + pairs = {"HTTPS_PROXY": url, "HTTP_PROXY": url, "NO_PROXY": "localhost,127.0.0.1"} + return [f"{key}={value}" for name, value in pairs.items() for key in (name, name.lower())] + + +def ensure(base_url: str) -> str: + """Create the internal network and an egress proxy allowing only the model + host; reuse them when they already match. Returns the allowed host.""" + host = model_host(base_url) + if docker_cli.docker("network", "inspect", NETWORK, check=False).returncode != 0: + docker_cli.docker("network", "create", "--internal", NETWORK) + current = docker_cli.docker( + "inspect", "--format", "{{.State.Running}} {{range .Config.Env}}{{.}} {{end}}", PROXY, + check=False, + ) + if current.returncode == 0 and current.stdout.startswith("true") and f"ALLOW_HOSTS={host} " in current.stdout + " ": + return host + docker_cli.remove_container(PROXY) + docker_cli.pull(PROXY_IMAGE, platform="linux/arm64" if _host_is_arm() else "linux/amd64") + docker_cli.docker( + "create", "--name", PROXY, "--network", "bridge", "--restart", "unless-stopped", + "-e", f"ALLOW_HOSTS={host}", "-e", "ALLOW_PORTS=443", "-e", f"PROXY_PORT={PROXY_PORT}", + PROXY_IMAGE, "python", "-u", "/egress_proxy.py", + ) + docker_cli.copy_to(PROXY, PROXY_SOURCE, "/egress_proxy.py") + docker_cli.docker("network", "connect", NETWORK, PROXY) + docker_cli.docker("start", PROXY) + return host + + +def _host_is_arm() -> bool: + arch = docker_cli.docker("info", "--format", "{{.Architecture}}", check=False).stdout.strip() + return arch in {"aarch64", "arm64"} + + +def denied_attempts(since: str | None = None) -> list[str]: + """Proxy log lines of refused connections (audit trail).""" + args = ["logs", PROXY] + (["--since", since] if since else []) + logs = docker_cli.docker(*args, check=False) + return [line for line in (logs.stdout + logs.stderr).splitlines() if " DENY " in line] diff --git a/benchmarks/swebench/code4me_swebench/settings.py b/benchmarks/swebench/code4me_swebench/settings.py new file mode 100644 index 0000000..20bf4ed --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/settings.py @@ -0,0 +1,156 @@ +"""Run settings and the agent configuration they produce.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import TYPE_CHECKING +from urllib.parse import urlparse + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + +# Every agent tool except ask_user: nobody answers questions in a benchmark. +DEFAULT_TOOLS = ( + "read_file", "create_file", "write_file", "replace_text", "edit_file", + "apply_patch", "delete_file", "move_file", "list_files", "glob_files", + "grep_files", "search_files", "run_command", "update_plan", +) + +# Paths inside the task container. +WORKSPACE = "/testbed" +RUNTIME_MOUNT = "/opt/code4me" +TASK_DIR = "/tmp/code4me" +OUT_DIR = f"{TASK_DIR}/out" + + +@dataclass(frozen=True) +class RunSettings: + # OpenCode Go model id. V4.1 Flash (2026-09-10) has a larger Go quota than + # V4 Flash (2026-07-31); runs before 2026-10-09 used deepseek-v4-flash. + model: str = "deepseek-v4.1-flash" + base_url: str = "https://opencode.ai/zen/go/v1" + # Name of the host environment variable holding the provider key. Only the + # name is written to disk; the value is passed to the container at exec time. + api_key_env: str = "OPENCODE_GO_API_KEY" + temperature: float | None = None + max_output_tokens: int | None = None + provider_timeout_s: float = 300.0 + # Hard wall-clock limit per model request (retried like a timeout); the read + # timeout alone never fires while DeepSeek sends keep-alive blank lines. + request_deadline_s: float = 300.0 + # The defaults below follow the SWE-rebench leaderboard's protocol (read from + # swe-rebench.com and its published trajectories on 2026-10-14), so results + # compare with its numbers for the same model and task window. + # Model calls per task; summarisation calls do not count. The leaderboard has + # had no step cap since Feb 2026 (its runs reached 492 steps): 500 is a cap + # that should practically never bind. + max_iterations: int = 500 + # Conversation tokens kept per model request (the agent calibrates its + # chars/4 estimate against the provider's counts). The leaderboard caps the + # context at 128k; past that its scaffold fails the task, ours compacts. + context_tokens: int = 128_000 + # The leaderboard's fixed per-command timeout. + command_timeout_s: float = 300.0 + max_command_timeout_s: float = 300.0 + max_command_output_bytes: int = 16_384 + # Wall-clock limit for one task; the agent is killed and its diff kept. A + # guard against hangs, not a budget (the leaderboard's longest run: 2.7 h). + task_timeout_s: float = 14_400.0 + # Programs the agent may not run (its commands denylist). Empty: any program + # in the task container can run; the container is the sandbox. + blocked_commands: tuple[str, ...] = () + tools: tuple[str, ...] = DEFAULT_TOOLS + # HarnessOptions overrides (empty = the agent's shipped defaults). + harness_options: dict[str, object] = field(default_factory=dict) + platform: str = "linux/amd64" + # Task containers reach the model API host and nothing else (no GitHub, + # PyPI, git remotes): agents otherwise look up the upstream fix. + network: str = "model-api-only" + # Extra top-level request fields for the model API, sent by our agent and by + # the mini-swe-agent baseline alike: e.g. OpenRouter's upstream pin. + extra_body: dict[str, object] = field(default_factory=dict) + + @property + def model_name_or_path(self) -> str: + return f"code4me2-agent__{self.model}" + + def as_dict(self) -> dict[str, object]: + data = asdict(self) + data["blocked_commands"] = list(self.blocked_commands) + data["tools"] = list(self.tools) + if not self.extra_body: + del data["extra_body"] # manifests of runs without it stay unchanged + return data + + +def agent_config(settings: RunSettings, *, session_id: str) -> dict[str, object]: + """The agent's config-file JSON (``AgentConfig.from_file`` format).""" + provider: dict[str, object] = { + "kind": "openai", + "base_url": settings.base_url, + "model": settings.model, + "api_key_env": settings.api_key_env, + "timeout_seconds": settings.provider_timeout_s, + "request_deadline_seconds": settings.request_deadline_s, + } + if settings.temperature is not None: + provider["temperature"] = settings.temperature + if settings.max_output_tokens is not None: + provider["max_output_tokens"] = settings.max_output_tokens + if settings.extra_body: + provider["extra_body"] = dict(settings.extra_body) + return { + "workspace_root": WORKSPACE, + "session_id": session_id, + # Headless: nobody answers, so a turn that ends announcing more work is + # continued (at most twice). IDE sessions never do this. + "autonomous": True, + "adapter": { + "name": "openai_compatible_react", + "max_iterations": settings.max_iterations, + "provider": provider, + "memory_window": { + "scope": "session", + "strategy": "token_window", + "max_messages": 1_000, + "max_tokens": settings.context_tokens, + }, + }, + "commands": { + "denylist": list(settings.blocked_commands), + "timeout_seconds": settings.command_timeout_s, + "max_timeout_seconds": settings.max_command_timeout_s, + "max_output_bytes": settings.max_command_output_bytes, + }, + "harness_options": dict(settings.harness_options), + "telemetry": { + "trace_path": f"{OUT_DIR}/trace.jsonl", + "raw_capture_enabled": True, + }, + } + + +OPENROUTER_HOST = "openrouter.ai" + + +def is_openrouter(base_url: str) -> bool: + host = (urlparse(base_url).hostname or "").lower() + return host == OPENROUTER_HOST or host.endswith("." + OPENROUTER_HOST) + + +def openrouter_routing(upstreams: Sequence[str]) -> dict[str, object]: + """OpenRouter fields that pin the upstream: each request is served by one of + ``upstreams`` or fails, never by another host or quantisation of the model.""" + return {"provider": {"only": list(upstreams), "allow_fallbacks": False, "require_parameters": True}} + + +def pinned_upstreams(extra_body: Mapping[str, object]) -> list[str]: + """The upstreams an OpenRouter routing block restricts requests to ([] = any).""" + provider = extra_body.get("provider") if extra_body else None + if not isinstance(provider, dict): + return [] + if provider.get("only"): + return [str(slug) for slug in provider["only"]] + if provider.get("order") and provider.get("allow_fallbacks") is False: + return [str(slug) for slug in provider["order"]] + return [] diff --git a/benchmarks/swebench/code4me_swebench/telemetry.py b/benchmarks/swebench/code4me_swebench/telemetry.py new file mode 100644 index 0000000..375de54 --- /dev/null +++ b/benchmarks/swebench/code4me_swebench/telemetry.py @@ -0,0 +1,138 @@ +"""Per-task summary of the agent's telemetry trace (``code4me.agent.event.v1``).""" + +from __future__ import annotations + +import json +from collections import Counter +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + +EDIT_TOOLS = frozenset({ + "create_file", "write_file", "replace_text", "edit_file", "apply_patch", + "delete_file", "move_file", +}) + + +def read_events(trace_path: Path) -> list[dict]: + if not trace_path.exists(): + return [] + events = [] + for line in trace_path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + events.append(json.loads(line)) + return events + + +def _response(event: dict) -> dict: + response = (event.get("raw_payload") or {}).get("response") + return response if isinstance(response, dict) else {} + + +def upstream(event: dict) -> str | None: + """The host that served a model call, when the API names it (OpenRouter's ``provider``).""" + name = _response(event).get("provider") + return name if isinstance(name, str) and name else None + + +def _cost(event: dict) -> float | None: + usage = _response(event).get("usage") + cost = usage.get("cost") if isinstance(usage, dict) else None + return float(cost) if isinstance(cost, (int, float)) and not isinstance(cost, bool) else None + + +def _cached_prompt_tokens(event: dict) -> int: + usage = _response(event).get("usage") + if not isinstance(usage, dict): + return 0 + # OpenCode routes to upstreams that report the cache in either or both of + # DeepSeek's and OpenAI's fields (sometimes one is present but null). + details = usage.get("prompt_tokens_details") or {} + return max(int(usage.get("prompt_cache_hit_tokens") or 0), int(details.get("cached_tokens") or 0)) + + +def summarize(trace_path: Path) -> dict: + events = read_events(trace_path) + by_type = Counter(event.get("event_type") for event in events) + summary: dict = { + "events": len(events), + "event_types": dict(sorted(by_type.items())), + "model_calls": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + "cached_prompt_tokens": 0, + "usage_estimated_calls": 0, + "model_time_s": 0.0, + "call_purposes": {}, + "tool_calls": {}, + "tool_failures": 0, + "tool_denials": 0, + "commands": {}, + "test_runs": 0, + "files_edited": [], + "run_status": None, + "stop_reason": None, + "duration_s": None, + "model_failures": 0, + "failed_side_calls": 0, + "context_compactions": 0, + # Provider-reported spend (OpenRouter returns it in usage); None if never reported. + "cost_usd": None, + "upstreams": {}, + } + purposes: Counter = Counter() + upstreams: Counter = Counter() + tools: Counter = Counter() + commands: Counter = Counter() + edited: set[str] = set() + for event in events: + kind = event.get("event_type") + payload = event.get("payload") or {} + metrics = event.get("metrics") or {} + if kind == "agent.model.completed": + summary["model_calls"] += 1 + summary["prompt_tokens"] += int(metrics.get("prompt_tokens") or 0) + summary["completion_tokens"] += int(metrics.get("completion_tokens") or 0) + summary["cached_prompt_tokens"] += _cached_prompt_tokens(event) + summary["usage_estimated_calls"] += bool(payload.get("usage_estimated")) + summary["model_time_s"] += float(metrics.get("duration_ms") or 0) / 1000 + purposes[payload.get("call_purpose") or "unknown"] += 1 + if upstream(event): + upstreams[upstream(event)] += 1 + if _cost(event) is not None: + summary["cost_usd"] = round((summary["cost_usd"] or 0.0) + _cost(event), 6) + elif kind in {"agent.tool.completed", "agent.tool.failed", "agent.tool.denied"}: + name = payload.get("tool_name") or "unknown" + tools[name] += 1 + if kind == "agent.tool.failed" or payload.get("status") == "failed": + summary["tool_failures"] += 1 + if kind == "agent.tool.denied" or payload.get("status") == "denied": + summary["tool_denials"] += 1 + argv = payload.get("argv") + if name == "run_command" and isinstance(argv, list) and argv: + commands[str(argv[0])] += 1 + # The agent flattens a parsed runner summary into test_framework/tests_*. + if payload.get("test_framework"): + summary["test_runs"] += 1 + path = payload.get("path") + if name in EDIT_TOOLS and kind == "agent.tool.completed" and isinstance(path, str): + edited.add(path) + elif kind == "agent.model.requested" and metrics.get("context_elided_units"): + summary["context_compactions"] += 1 + elif kind == "agent.adapter.loop_failed" and payload.get("call_purpose") == "turn": + summary["model_failures"] += 1 + elif kind == "agent.response.completed": + summary["stop_reason"] = payload.get("stop_reason") + elif kind == "agent.run.completed": + summary["run_status"] = payload.get("status") + summary["failed_side_calls"] += int(metrics.get("failed_side_calls") or 0) + duration_ms = metrics.get("duration_ms") + summary["duration_s"] = round(float(duration_ms) / 1000, 3) if duration_ms else None + summary["model_time_s"] = round(summary["model_time_s"], 3) + summary["call_purposes"] = dict(purposes.most_common()) + summary["upstreams"] = dict(upstreams.most_common()) + summary["tool_calls"] = dict(tools.most_common()) + summary["commands"] = dict(commands.most_common()) + summary["files_edited"] = sorted(edited) + return summary diff --git a/benchmarks/swebench/mini_baseline/opencode_model.py b/benchmarks/swebench/mini_baseline/opencode_model.py new file mode 100644 index 0000000..41692ff --- /dev/null +++ b/benchmarks/swebench/mini_baseline/opencode_model.py @@ -0,0 +1,20 @@ +"""mini-swe-agent model class for OpenCode (Zen/Go) endpoints. + +OpenCode rejects requests without ``x-opencode-session`` and asks for a stable id +per conversation. mini-swe-agent builds one model per task instance, so a fresh +id per instance gives one session per task. Loaded by ``mini-extra`` through +``--model-class opencode_model.OpenCodeModel`` with this folder on PYTHONPATH; +it imports minisweagent, so it only runs in the baseline's own venv. +""" + +import uuid + +from minisweagent.models.litellm_model import LitellmModel + + +class OpenCodeModel(LitellmModel): + def __init__(self, **kwargs): + super().__init__(**kwargs) + headers = dict(self.config.model_kwargs.get("extra_headers") or {}) + headers["x-opencode-session"] = f"msa-{uuid.uuid4().hex}" + self.config.model_kwargs = {**self.config.model_kwargs, "extra_headers": headers} diff --git a/benchmarks/swebench/pyproject.toml b/benchmarks/swebench/pyproject.toml new file mode 100644 index 0000000..934fc8e --- /dev/null +++ b/benchmarks/swebench/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "code4me-swebench" +version = "0.1.0" +description = "Run the packaged Code4Me agent on SWE-bench tasks and collect its telemetry" +requires-python = ">=3.11" +dependencies = [ + # Official evaluation harness (instance images, grading). The agent itself + # is not a host dependency: it runs inside each task container. + "swebench==5.0.2", +] + +[project.optional-dependencies] +# The agent from this checkout, so tests can drive driver.py end to end. +test = ["pytest>=8", "code4me2-agent"] + +[tool.uv.sources] +code4me2-agent = { path = "../..", editable = true } + +[project.scripts] +code4me-swebench = "code4me_swebench.cli:main" + +[tool.setuptools.packages.find] +include = ["code4me_swebench*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/benchmarks/swebench/references.json b/benchmarks/swebench/references.json new file mode 100644 index 0000000..e12113a --- /dev/null +++ b/benchmarks/swebench/references.json @@ -0,0 +1,179 @@ +{ + "about": "Published SWE-rebench leaderboard results (swe-rebench.com) for comparison with our runs on the same task windows. Read from the site's page data on 2026-10-14 (site last built 2026-07-28). Resolved rate and pass@5 are percentages over 5 runs of the leaderboard's own scaffold; cost uses the leaderboard's unpublished price table. Window keys are [start, end) by task creation date.", + "references": { + "deepseek-v4-flash-high": { + "label": "DeepSeek-V4 Flash [high] (tools)", + "model_release": "2026-04-24", + "source": "https://swe-rebench.com/", + "retrieved": "2026-10-14", + "windows": { + "2026-03-01..2026-04-01": { + "tasks": 46, + "resolved_rate": 36.09, + "sem": 1.63, + "pass_at_5": 56.5, + "cost_per_problem": 0.071, + "tokens_per_problem": 3060000, + "cached_share": 95.1, + "contamination_flag": true + }, + "2026-03-01..2026-05-15": { + "tasks": 110, + "resolved_rate": 38.36, + "sem": 0.97, + "pass_at_5": 57.3, + "cost_per_problem": 0.069, + "tokens_per_problem": 3000000, + "cached_share": 95.5, + "contamination_flag": true + }, + "2026-04-01..2026-05-15": { + "tasks": 64, + "resolved_rate": 40.0, + "sem": 2.13, + "pass_at_5": 57.8, + "cost_per_problem": 0.068, + "tokens_per_problem": 2950000, + "cached_share": 95.8, + "contamination_flag": true + }, + "2026-04-15..2026-05-15": { + "tasks": 39, + "resolved_rate": 37.95, + "sem": 2.49, + "pass_at_5": 59.0, + "cost_per_problem": 0.073, + "tokens_per_problem": 3180000, + "cached_share": 95.8, + "contamination_flag": true + }, + "2026-05-01..2026-05-15": { + "tasks": 21, + "resolved_rate": 40.0, + "sem": 2.43, + "pass_at_5": 47.6, + "cost_per_problem": 0.086, + "tokens_per_problem": 3790000, + "cached_share": 96.3, + "contamination_flag": false + } + } + }, + "deepseek-v4-pro-high": { + "label": "DeepSeek-V4 Pro [high] (tools)", + "model_release": "2026-04-24", + "source": "https://swe-rebench.com/", + "retrieved": "2026-10-14", + "windows": { + "2026-03-01..2026-04-01": { + "tasks": 46, + "resolved_rate": 46.72, + "sem": 1.46, + "pass_at_5": 65.2, + "cost_per_problem": 0.192, + "tokens_per_problem": 2240000, + "cached_share": 79.6, + "contamination_flag": true + }, + "2026-03-01..2026-05-15": { + "tasks": 110, + "resolved_rate": 42.7, + "sem": 1.29, + "pass_at_5": 61.8, + "cost_per_problem": 0.217, + "tokens_per_problem": 2250000, + "cached_share": 76.9, + "contamination_flag": true + }, + "2026-04-01..2026-05-15": { + "tasks": 64, + "resolved_rate": 39.81, + "sem": 2.43, + "pass_at_5": 59.4, + "cost_per_problem": 0.235, + "tokens_per_problem": 2250000, + "cached_share": 75.0, + "contamination_flag": true + }, + "2026-04-15..2026-05-15": { + "tasks": 39, + "resolved_rate": 37.62, + "sem": 2.21, + "pass_at_5": 59.0, + "cost_per_problem": 0.23, + "tokens_per_problem": 2370000, + "cached_share": 76.7, + "contamination_flag": true + }, + "2026-05-01..2026-05-15": { + "tasks": 21, + "resolved_rate": 36.19, + "sem": 1.17, + "pass_at_5": 52.4, + "cost_per_problem": 0.248, + "tokens_per_problem": 2690000, + "cached_share": 76.8, + "contamination_flag": false + } + } + }, + "mimo-v2.5-pro": { + "label": "MiMo V2.5 Pro (tools)", + "model_release": "2026-03-22", + "source": "https://swe-rebench.com/", + "retrieved": "2026-10-14", + "windows": { + "2026-03-01..2026-04-01": { + "tasks": 46, + "resolved_rate": 43.91, + "sem": 1.06, + "pass_at_5": 65.2, + "cost_per_problem": 0.132, + "tokens_per_problem": 2600000, + "cached_share": 88.1, + "contamination_flag": true + }, + "2026-03-01..2026-05-15": { + "tasks": 110, + "resolved_rate": 42.36, + "sem": 0.84, + "pass_at_5": 61.8, + "cost_per_problem": 0.125, + "tokens_per_problem": 2590000, + "cached_share": 88.6, + "contamination_flag": true + }, + "2026-04-01..2026-05-15": { + "tasks": 64, + "resolved_rate": 41.25, + "sem": 1.06, + "pass_at_5": 59.4, + "cost_per_problem": 0.119, + "tokens_per_problem": 2580000, + "cached_share": 88.9, + "contamination_flag": false + }, + "2026-04-15..2026-05-15": { + "tasks": 39, + "resolved_rate": 39.49, + "sem": 1.31, + "pass_at_5": 59.0, + "cost_per_problem": 0.118, + "tokens_per_problem": 2520000, + "cached_share": 88.9, + "contamination_flag": false + }, + "2026-05-01..2026-05-15": { + "tasks": 21, + "resolved_rate": 39.05, + "sem": 2.78, + "pass_at_5": 61.9, + "cost_per_problem": 0.119, + "tokens_per_problem": 2670000, + "cached_share": 90.0, + "contamination_flag": false + } + } + } + } +} diff --git a/benchmarks/swebench/subsets/rebench-2026_03-0301-0401.txt b/benchmarks/swebench/subsets/rebench-2026_03-0301-0401.txt new file mode 100644 index 0000000..9d12295 --- /dev/null +++ b/benchmarks/swebench/subsets/rebench-2026_03-0301-0401.txt @@ -0,0 +1,47 @@ +# 46 instances of nebius/SWE-rebench-leaderboard (2026_03), every task created in [2026-03-01, 2026-04-01) (a SWE-rebench leaderboard window), dataset revision 34d5a58864acf91613740a09ec5d205228dcfa39. Generated by `code4me-swebench select --window`. +keras-team__keras-22316 +pallets__click-3239 +tobymao__sqlglot-7187 +keras-team__keras-22330 +tox-dev__tox-3846 +containers__ramalama-2487 +astronomy-commons__hats-648 +hiero-ledger__hiero-sdk-python-1914_interface +zauberzeug__nicegui-5858 +getmoto__moto-9841 +conan-io__conan-19735_interface +koxudaxi__datamodel-code-generator-3046 +alibaba__opensandbox-426 +destine-climate-dt__aqua-2768 +pycqa__isort-2491 +conan-io__conan-19750 +sqlfluff__sqlfluff-7615 +dagster-io__dagster-33605 +sqlfluff__sqlfluff-7617 +astropy__astropy-19438 +sqlfluff__sqlfluff-7622 +scikit-learn__scikit-learn-33565 +psd-tools__psd-tools-593 +con__nwb2bids-347_interface +vprusso__toqito-1484_interface +stanfordnlp__dspy-9478_interface +hiero-ledger__hiero-sdk-python-2007_interface +idaholab__montepy-933_interface +genericmappingtools__pygmt-4463 +harumiweb__exstruct-113 +sunpy__sunpy-8548_interface +pgmpy__pgmpy-3137 +getmoto__moto-9903 +pandas-dev__pandas-64796 +pandas-dev__pandas-64797 +kubeflow__pipelines-13112 +pandas-dev__pandas-64816 +pyinfra-dev__pyinfra-1630 +scrapy__scrapy-7351_interface +delgan__loguru-1451 +holoviz__param-1117 +marshmallow-code__marshmallow-2925_interface +tox-dev__tox-3904 +oemof__tespy-921 +zauberzeug__nicegui-5914 +ucfopen__canvasapi-716 diff --git a/benchmarks/swebench/subsets/rebench-2026_03-0301-0515.txt b/benchmarks/swebench/subsets/rebench-2026_03-0301-0515.txt new file mode 100644 index 0000000..b6f9a57 --- /dev/null +++ b/benchmarks/swebench/subsets/rebench-2026_03-0301-0515.txt @@ -0,0 +1,111 @@ +# 110 instances of nebius/SWE-rebench-leaderboard (2026_03), every task created in [2026-03-01, 2026-05-15) (a SWE-rebench leaderboard window), dataset revision 34d5a58864acf91613740a09ec5d205228dcfa39. Generated by `code4me-swebench select --window`. +keras-team__keras-22316 +pallets__click-3239 +tobymao__sqlglot-7187 +keras-team__keras-22330 +tox-dev__tox-3846 +containers__ramalama-2487 +astronomy-commons__hats-648 +hiero-ledger__hiero-sdk-python-1914_interface +zauberzeug__nicegui-5858 +getmoto__moto-9841 +conan-io__conan-19735_interface +koxudaxi__datamodel-code-generator-3046 +alibaba__opensandbox-426 +destine-climate-dt__aqua-2768 +pycqa__isort-2491 +conan-io__conan-19750 +sqlfluff__sqlfluff-7615 +dagster-io__dagster-33605 +sqlfluff__sqlfluff-7617 +astropy__astropy-19438 +sqlfluff__sqlfluff-7622 +scikit-learn__scikit-learn-33565 +psd-tools__psd-tools-593 +con__nwb2bids-347_interface +vprusso__toqito-1484_interface +stanfordnlp__dspy-9478_interface +hiero-ledger__hiero-sdk-python-2007_interface +idaholab__montepy-933_interface +genericmappingtools__pygmt-4463 +harumiweb__exstruct-113 +sunpy__sunpy-8548_interface +pgmpy__pgmpy-3137 +getmoto__moto-9903 +pandas-dev__pandas-64796 +pandas-dev__pandas-64797 +kubeflow__pipelines-13112 +pandas-dev__pandas-64816 +pyinfra-dev__pyinfra-1630 +scrapy__scrapy-7351_interface +delgan__loguru-1451 +holoviz__param-1117 +marshmallow-code__marshmallow-2925_interface +tox-dev__tox-3904 +oemof__tespy-921 +zauberzeug__nicegui-5914 +ucfopen__canvasapi-716 +line__line-bot-sdk-python-981_interface +pdm-project__pdm-3759 +python-scim__scim2-models-139_interface +meltano__meltano-9929 +koxudaxi__datamodel-code-generator-3070 +koxudaxi__datamodel-code-generator-3071 +tobymao__sqlglot-7457 +cpp-linter__cpp-linter-hooks-206_interface +huggingface__huggingface_hub-4056 +keras-team__keras-22642 +azure-samples__azure-search-openai-demo-3025 +graphistry__pygraphistry-1107_interface +tobymao__sqlglot-7479 +pypa__build-1027 +meltano__meltano-9950 +unit8co__darts-3065 +agentscope-ai__qwenpaw-3278 +nesquena__hermes-webui-330_interface +spack__spack-52244 +pypsa__pypsa-1653 +deltares__hydrolib-core-1049_interface +microsoft__qcodes-8039 +pennylaneai__pennylane-9298 +pipecat-ai__pipecat-4283 +pyinfra-dev__pyinfra-1665 +ultraplot__ultraplot-696 +pypa__twine-1309 +celestoai__smolvm-172 +mempalace__mempalace-1004 +pyinfra-dev__pyinfra-1679_interface +schemathesis__schemathesis-3778 +agronholm__anyio-1121 +vprusso__toqito-1538 +sigma67__ytmusicapi-909_interface +zarr-developers__virtualizarr-979 +databricks__dbt-databricks-1428 +reframe-hpc__reframe-3660_interface +apache__iceberg-python-3295 +alibaba__opensandbox-816 +beever-ai__beever-atlas-102 +astronomy-commons__lsdb-1349 +hkuds__openharness-217 +copier-org__copier-2646 +tox-dev__tox-3931 +pallets-eco__wtforms-892_interface +graphistry__pygraphistry-1277 +schemathesis__schemathesis-3933 +python-wheel-build__fromager-1106 +raullenchai__rapid-mlx-227 +raullenchai__rapid-mlx-228 +agronholm__anyio-1134 +itsdnns__docsight-437 +python-wheel-build__fromager-1124 +olofk__fusesoc-776_interface +nesquena__hermes-webui-1818 +raullenchai__rapid-mlx-289 +youssofal__mtplx-21 +macbre__sql-metadata-625_interface +raullenchai__rapid-mlx-341_interface +geopython__pygeoapi-2338 +nesquena__hermes-webui-2056 +cs-si__eodag-2176 +aallan__vera-662_interface +schemathesis__schemathesis-4087 diff --git a/benchmarks/swebench/subsets/rebench-2026_03-0401-0515.txt b/benchmarks/swebench/subsets/rebench-2026_03-0401-0515.txt new file mode 100644 index 0000000..8c4ed0a --- /dev/null +++ b/benchmarks/swebench/subsets/rebench-2026_03-0401-0515.txt @@ -0,0 +1,65 @@ +# 64 instances of nebius/SWE-rebench-leaderboard (2026_03), every task created in [2026-04-01, 2026-05-15) (a SWE-rebench leaderboard window), dataset revision 34d5a58864acf91613740a09ec5d205228dcfa39. Generated by `code4me-swebench select --window`. +line__line-bot-sdk-python-981_interface +pdm-project__pdm-3759 +python-scim__scim2-models-139_interface +meltano__meltano-9929 +koxudaxi__datamodel-code-generator-3070 +koxudaxi__datamodel-code-generator-3071 +tobymao__sqlglot-7457 +cpp-linter__cpp-linter-hooks-206_interface +huggingface__huggingface_hub-4056 +keras-team__keras-22642 +azure-samples__azure-search-openai-demo-3025 +graphistry__pygraphistry-1107_interface +tobymao__sqlglot-7479 +pypa__build-1027 +meltano__meltano-9950 +unit8co__darts-3065 +agentscope-ai__qwenpaw-3278 +nesquena__hermes-webui-330_interface +spack__spack-52244 +pypsa__pypsa-1653 +deltares__hydrolib-core-1049_interface +microsoft__qcodes-8039 +pennylaneai__pennylane-9298 +pipecat-ai__pipecat-4283 +pyinfra-dev__pyinfra-1665 +ultraplot__ultraplot-696 +pypa__twine-1309 +celestoai__smolvm-172 +mempalace__mempalace-1004 +pyinfra-dev__pyinfra-1679_interface +schemathesis__schemathesis-3778 +agronholm__anyio-1121 +vprusso__toqito-1538 +sigma67__ytmusicapi-909_interface +zarr-developers__virtualizarr-979 +databricks__dbt-databricks-1428 +reframe-hpc__reframe-3660_interface +apache__iceberg-python-3295 +alibaba__opensandbox-816 +beever-ai__beever-atlas-102 +astronomy-commons__lsdb-1349 +hkuds__openharness-217 +copier-org__copier-2646 +tox-dev__tox-3931 +pallets-eco__wtforms-892_interface +graphistry__pygraphistry-1277 +schemathesis__schemathesis-3933 +python-wheel-build__fromager-1106 +raullenchai__rapid-mlx-227 +raullenchai__rapid-mlx-228 +agronholm__anyio-1134 +itsdnns__docsight-437 +python-wheel-build__fromager-1124 +olofk__fusesoc-776_interface +nesquena__hermes-webui-1818 +raullenchai__rapid-mlx-289 +youssofal__mtplx-21 +macbre__sql-metadata-625_interface +raullenchai__rapid-mlx-341_interface +geopython__pygeoapi-2338 +nesquena__hermes-webui-2056 +cs-si__eodag-2176 +aallan__vera-662_interface +schemathesis__schemathesis-4087 diff --git a/benchmarks/swebench/subsets/rebench-2026_03-0415-0515.txt b/benchmarks/swebench/subsets/rebench-2026_03-0415-0515.txt new file mode 100644 index 0000000..7248f65 --- /dev/null +++ b/benchmarks/swebench/subsets/rebench-2026_03-0415-0515.txt @@ -0,0 +1,40 @@ +# 39 instances of nebius/SWE-rebench-leaderboard (2026_03), every task created in [2026-04-15, 2026-05-15) (a SWE-rebench leaderboard window), dataset revision 34d5a58864acf91613740a09ec5d205228dcfa39. Generated by `code4me-swebench select --window`. +ultraplot__ultraplot-696 +pypa__twine-1309 +celestoai__smolvm-172 +mempalace__mempalace-1004 +pyinfra-dev__pyinfra-1679_interface +schemathesis__schemathesis-3778 +agronholm__anyio-1121 +vprusso__toqito-1538 +sigma67__ytmusicapi-909_interface +zarr-developers__virtualizarr-979 +databricks__dbt-databricks-1428 +reframe-hpc__reframe-3660_interface +apache__iceberg-python-3295 +alibaba__opensandbox-816 +beever-ai__beever-atlas-102 +astronomy-commons__lsdb-1349 +hkuds__openharness-217 +copier-org__copier-2646 +tox-dev__tox-3931 +pallets-eco__wtforms-892_interface +graphistry__pygraphistry-1277 +schemathesis__schemathesis-3933 +python-wheel-build__fromager-1106 +raullenchai__rapid-mlx-227 +raullenchai__rapid-mlx-228 +agronholm__anyio-1134 +itsdnns__docsight-437 +python-wheel-build__fromager-1124 +olofk__fusesoc-776_interface +nesquena__hermes-webui-1818 +raullenchai__rapid-mlx-289 +youssofal__mtplx-21 +macbre__sql-metadata-625_interface +raullenchai__rapid-mlx-341_interface +geopython__pygeoapi-2338 +nesquena__hermes-webui-2056 +cs-si__eodag-2176 +aallan__vera-662_interface +schemathesis__schemathesis-4087 diff --git a/benchmarks/swebench/subsets/rebench-2026_03-0501-0515.txt b/benchmarks/swebench/subsets/rebench-2026_03-0501-0515.txt new file mode 100644 index 0000000..a1403ea --- /dev/null +++ b/benchmarks/swebench/subsets/rebench-2026_03-0501-0515.txt @@ -0,0 +1,22 @@ +# 21 instances of nebius/SWE-rebench-leaderboard (2026_03), every task created in [2026-05-01, 2026-05-15) (a SWE-rebench leaderboard window), dataset revision 34d5a58864acf91613740a09ec5d205228dcfa39. Generated by `code4me-swebench select --window`. +tox-dev__tox-3931 +pallets-eco__wtforms-892_interface +graphistry__pygraphistry-1277 +schemathesis__schemathesis-3933 +python-wheel-build__fromager-1106 +raullenchai__rapid-mlx-227 +raullenchai__rapid-mlx-228 +agronholm__anyio-1134 +itsdnns__docsight-437 +python-wheel-build__fromager-1124 +olofk__fusesoc-776_interface +nesquena__hermes-webui-1818 +raullenchai__rapid-mlx-289 +youssofal__mtplx-21 +macbre__sql-metadata-625_interface +raullenchai__rapid-mlx-341_interface +geopython__pygeoapi-2338 +nesquena__hermes-webui-2056 +cs-si__eodag-2176 +aallan__vera-662_interface +schemathesis__schemathesis-4087 diff --git a/benchmarks/swebench/subsets/rebench-2026_03-newest-50.txt b/benchmarks/swebench/subsets/rebench-2026_03-newest-50.txt new file mode 100644 index 0000000..dbaf58c --- /dev/null +++ b/benchmarks/swebench/subsets/rebench-2026_03-newest-50.txt @@ -0,0 +1,51 @@ +# 50 instances of nebius/SWE-rebench-leaderboard (2026_03), the newest by created_at (2026-04-10 .. 2026-05-12). Generated by `code4me-swebench select`. +aallan__vera-662_interface +agentscope-ai__qwenpaw-3278 +agronholm__anyio-1121 +agronholm__anyio-1134 +alibaba__opensandbox-816 +apache__iceberg-python-3295 +astronomy-commons__lsdb-1349 +beever-ai__beever-atlas-102 +celestoai__smolvm-172 +copier-org__copier-2646 +cs-si__eodag-2176 +databricks__dbt-databricks-1428 +deltares__hydrolib-core-1049_interface +geopython__pygeoapi-2338 +graphistry__pygraphistry-1277 +hkuds__openharness-217 +itsdnns__docsight-437 +macbre__sql-metadata-625_interface +meltano__meltano-9950 +mempalace__mempalace-1004 +microsoft__qcodes-8039 +nesquena__hermes-webui-1818 +nesquena__hermes-webui-2056 +nesquena__hermes-webui-330_interface +olofk__fusesoc-776_interface +pallets-eco__wtforms-892_interface +pennylaneai__pennylane-9298 +pipecat-ai__pipecat-4283 +pyinfra-dev__pyinfra-1665 +pyinfra-dev__pyinfra-1679_interface +pypa__twine-1309 +pypsa__pypsa-1653 +python-wheel-build__fromager-1106 +python-wheel-build__fromager-1124 +raullenchai__rapid-mlx-227 +raullenchai__rapid-mlx-228 +raullenchai__rapid-mlx-289 +raullenchai__rapid-mlx-341_interface +reframe-hpc__reframe-3660_interface +schemathesis__schemathesis-3778 +schemathesis__schemathesis-3933 +schemathesis__schemathesis-4087 +sigma67__ytmusicapi-909_interface +spack__spack-52244 +tox-dev__tox-3931 +ultraplot__ultraplot-696 +unit8co__darts-3065 +vprusso__toqito-1538 +youssofal__mtplx-21 +zarr-developers__virtualizarr-979 diff --git a/benchmarks/swebench/subsets/verified-mini.txt b/benchmarks/swebench/subsets/verified-mini.txt new file mode 100644 index 0000000..c52e1bf --- /dev/null +++ b/benchmarks/swebench/subsets/verified-mini.txt @@ -0,0 +1,53 @@ +# SWE-bench Verified Mini: 50 instances (django, sphinx) chosen by its author to match +# Verified's resolve-rate and difficulty distribution with ~5% of the image storage. +# Source: https://huggingface.co/datasets/MariusHobbhahn/swe-bench-verified-mini +django__django-11790 +django__django-11815 +django__django-11848 +django__django-11880 +django__django-11885 +django__django-11951 +django__django-11964 +django__django-11999 +django__django-12039 +django__django-12050 +django__django-12143 +django__django-12155 +django__django-12193 +django__django-12209 +django__django-12262 +django__django-12273 +django__django-12276 +django__django-12304 +django__django-12308 +django__django-12325 +django__django-12406 +django__django-12708 +django__django-12713 +django__django-12774 +django__django-9296 +sphinx-doc__sphinx-10323 +sphinx-doc__sphinx-10435 +sphinx-doc__sphinx-10466 +sphinx-doc__sphinx-10673 +sphinx-doc__sphinx-11510 +sphinx-doc__sphinx-7590 +sphinx-doc__sphinx-7748 +sphinx-doc__sphinx-7757 +sphinx-doc__sphinx-7985 +sphinx-doc__sphinx-8035 +sphinx-doc__sphinx-8056 +sphinx-doc__sphinx-8265 +sphinx-doc__sphinx-8269 +sphinx-doc__sphinx-8475 +sphinx-doc__sphinx-8548 +sphinx-doc__sphinx-8551 +sphinx-doc__sphinx-8638 +sphinx-doc__sphinx-8721 +sphinx-doc__sphinx-9229 +sphinx-doc__sphinx-9230 +sphinx-doc__sphinx-9281 +sphinx-doc__sphinx-9320 +sphinx-doc__sphinx-9367 +sphinx-doc__sphinx-9461 +sphinx-doc__sphinx-9698 diff --git a/benchmarks/swebench/subsets/verified-stratified-20.txt b/benchmarks/swebench/subsets/verified-stratified-20.txt new file mode 100644 index 0000000..74c5c1a --- /dev/null +++ b/benchmarks/swebench/subsets/verified-stratified-20.txt @@ -0,0 +1,21 @@ +# 20 instances of SWE-bench/SWE-bench_Verified, stratified by repo x difficulty (largest remainder), seed 0. Generated by `code4me-swebench select`. +astropy__astropy-8707 +django__django-11206 +django__django-12858 +django__django-13512 +django__django-13786 +django__django-14011 +django__django-14534 +django__django-15104 +django__django-15315 +django__django-16082 +django__django-16938 +matplotlib__matplotlib-24149 +matplotlib__matplotlib-25960 +pydata__xarray-6721 +scikit-learn__scikit-learn-13124 +sphinx-doc__sphinx-7454 +sphinx-doc__sphinx-9673 +sympy__sympy-13798 +sympy__sympy-18211 +sympy__sympy-24562 diff --git a/benchmarks/swebench/subsets/verified-stratified-50.txt b/benchmarks/swebench/subsets/verified-stratified-50.txt new file mode 100644 index 0000000..dae74fa --- /dev/null +++ b/benchmarks/swebench/subsets/verified-stratified-50.txt @@ -0,0 +1,51 @@ +# 50 instances of SWE-bench/SWE-bench_Verified, stratified by repo x difficulty (largest remainder), seed 0. Generated by `code4me-swebench select`. +astropy__astropy-14182 +astropy__astropy-8707 +django__django-11138 +django__django-11433 +django__django-11820 +django__django-12419 +django__django-12858 +django__django-13112 +django__django-13121 +django__django-13401 +django__django-13410 +django__django-13658 +django__django-14053 +django__django-14122 +django__django-14170 +django__django-14631 +django__django-14725 +django__django-15277 +django__django-15467 +django__django-16139 +django__django-16315 +django__django-16612 +django__django-16950 +django__django-17087 +django__django-7530 +matplotlib__matplotlib-20676 +matplotlib__matplotlib-22865 +matplotlib__matplotlib-24026 +matplotlib__matplotlib-25332 +psf__requests-1142 +pydata__xarray-4356 +pydata__xarray-7233 +pydata__xarray-7393 +pylint-dev__pylint-6528 +pytest-dev__pytest-5631 +pytest-dev__pytest-7432 +scikit-learn__scikit-learn-14496 +scikit-learn__scikit-learn-14710 +scikit-learn__scikit-learn-25747 +sphinx-doc__sphinx-7985 +sphinx-doc__sphinx-9230 +sphinx-doc__sphinx-9281 +sphinx-doc__sphinx-9658 +sympy__sympy-11618 +sympy__sympy-13091 +sympy__sympy-13615 +sympy__sympy-14248 +sympy__sympy-17139 +sympy__sympy-23262 +sympy__sympy-24539 diff --git a/benchmarks/swebench/tests/test_check.py b/benchmarks/swebench/tests/test_check.py new file mode 100644 index 0000000..a0c1943 --- /dev/null +++ b/benchmarks/swebench/tests/test_check.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import json + +from code4me_swebench import check + + +def _event(seq, kind, payload=None, **extra): + return {"schema_version": check.SCHEMA, "event_id": f"e{seq}", "event_type": kind, + "timestamp": f"2026-10-09T00:00:{seq:02d}Z", "sequence": seq, "run_id": "r", + "session_id": "s", "parent_event_id": None, "payload": payload or {}, **extra} + + +def _write(path, events): + path.write_text("".join(json.dumps(event) + "\n" for event in events)) + return path + + +def _good(): + usage = {"prompt_tokens": 10, "completion_tokens": 2} + return [ + _event(1, "agent.run.started"), + _event(2, "agent.model.requested"), + _event(3, "agent.model.completed", metrics=usage, raw_payload={"response": {"usage": usage}}), + _event(4, "agent.tool.called", {"tool_call_id": "t1"}), + _event(5, "agent.tool.completed", {"tool_call_id": "t1"}), + _event(6, "agent.model.requested", {"call_purpose": "self_review"}), + _event(7, "agent.response.completed"), + _event(8, "agent.run.completed", {"status": "completed"}, metrics={"failed_side_calls": 1}), + ] + + +def test_a_complete_trace_has_no_problems(tmp_path): + assert check.check_trace(_write(tmp_path / "t.jsonl", _good()), secret="sk-secret") == [] + + +def test_unanswered_requests_unresolved_tools_and_token_drift_are_reported(tmp_path): + events = _good() + events[2]["metrics"] = {"prompt_tokens": 11, "completion_tokens": 2} + events[7]["metrics"] = {} # the failed review is no longer reported + del events[4] # the tool outcome + for seq, event in enumerate(events, start=1): + event["sequence"] = seq + problems = check.check_trace(_write(tmp_path / "t.jsonl", events)) + assert any("model requests" in p for p in problems) + assert any("tool calls without an outcome" in p for p in problems) + assert any("differ from provider usage" in p for p in problems) + + +def test_a_leaked_key_and_gaps_in_sequence_are_reported(tmp_path): + events = _good() + events[3]["payload"]["stdout"] = "OPENCODE=sk-secret" + events[5]["sequence"] = 99 + problems = check.check_trace(_write(tmp_path / "t.jsonl", events), secret="sk-secret") + assert "provider key found in trace" in problems + assert any("sequence" in p for p in problems) diff --git a/benchmarks/swebench/tests/test_driver.py b/benchmarks/swebench/tests/test_driver.py new file mode 100644 index 0000000..9301656 --- /dev/null +++ b/benchmarks/swebench/tests/test_driver.py @@ -0,0 +1,145 @@ +"""driver.py drives the real agent core; a scripted provider stands in for the model.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from code4me_swebench import driver, telemetry +from code4me_swebench.settings import DEFAULT_TOOLS, RunSettings, agent_config + + +def _call(call_id: str, name: str, **arguments: object) -> dict: + return {"id": call_id, "name": name, "arguments": arguments} + + +def _task(tmp_path: Path, workspace: Path, script: list[dict]) -> Path: + config = agent_config(RunSettings(max_iterations=6), session_id="swebench-test") + config["workspace_root"] = str(workspace) + config["telemetry"]["trace_path"] = str(tmp_path / "out" / "trace.jsonl") + config["adapter"]["fake_provider"] = {"enabled": True, "script": script} + # The real verification nudge needs a model turn the script does not have. + config["harness_options"] = {"verify_on_stop": False, "self_review": False} + task = { + "instance_id": "demo__demo-1", + "run_id": "run-demo-1", + "out_dir": str(tmp_path / "out"), + "prompt": "Fix the constant in a.py.", + "agent_config": config, + "allowed_tools": list(DEFAULT_TOOLS), + } + path = tmp_path / "task.json" + path.write_text(json.dumps(task)) + return path + + +def test_agent_config_round_trips_through_the_agent_loader(tmp_path): + from code4me2_agent.config import AgentConfig + + settings = RunSettings(model="deepseek-v4-flash", max_iterations=77, context_tokens=50_000) + path = tmp_path / "agent-config.json" + path.write_text(json.dumps(agent_config(settings, session_id="s-1"))) + config = AgentConfig.from_file(path) + + provider = config.adapter.provider + assert (provider.kind, provider.base_url, provider.model) == ( + "openai", "https://opencode.ai/zen/go/v1", "deepseek-v4-flash") + assert provider.api_key_env == "OPENCODE_GO_API_KEY" + assert provider.request_deadline_seconds == 300.0 + assert config.adapter.name == "openai_compatible_react" + assert config.adapter.max_iterations == 77 + assert config.adapter.memory_window.max_tokens == 50_000 + assert config.session_id == "s-1" + assert str(config.workspace_root) == "/testbed" + # (/tmp resolves to /private/tmp on macOS; it stays /tmp in the container.) + assert config.trace_path == Path("/tmp/code4me/out/trace.jsonl").resolve() + assert config.raw_capture_enabled is True + assert config.autonomous is True + assert config.commands.blocked_commands == [] + + +def test_driver_runs_the_agent_and_writes_result_and_trace(tmp_path): + workspace = tmp_path / "repo" + workspace.mkdir() + (workspace / "a.py").write_text("VALUE = 1\n") + script = [ + {"tool_calls": [_call("c1", "read_file", path="a.py")], + "usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110}}, + {"tool_calls": [_call("c2", "replace_text", path="a.py", old_text="VALUE = 1", + new_text="VALUE = 2")]}, + {"final_answer": "Changed VALUE to 2."}, + ] + + exit_code = driver.main(["driver.py", str(_task(tmp_path, workspace, script))]) + + result = json.loads((tmp_path / "out" / "result.json").read_text()) + assert exit_code == 0, result.get("traceback") + assert result["status"] == "completed" + assert result["run_id"] == "run-demo-1" + assert "VALUE to 2" in result["final_response"] + assert (workspace / "a.py").read_text() == "VALUE = 2\n" + + summary = telemetry.summarize(tmp_path / "out" / "trace.jsonl") + assert summary["model_calls"] == 3 + assert summary["prompt_tokens"] >= 100 + assert summary["tool_calls"] == {"read_file": 1, "replace_text": 1} + assert summary["files_edited"] == ["a.py"] + assert summary["run_status"] == "completed" + + +def test_driver_keeps_ask_user_out_of_the_tool_set(tmp_path): + workspace = tmp_path / "repo" + workspace.mkdir() + task_path = _task(tmp_path, workspace, [{"final_answer": "done"}]) + task = json.loads(task_path.read_text()) + + config = driver.build_agent_config(task) + + assert "ask_user" not in config.allowed_tools + assert config.approval_policy == "auto" + assert config.store_agent_content is True + + +def test_driver_records_failures_instead_of_raising(tmp_path): + workspace = tmp_path / "repo" + workspace.mkdir() + task_path = _task(tmp_path, workspace, []) + task = json.loads(task_path.read_text()) + del task["prompt"] + task_path.write_text(json.dumps(task)) + + exit_code = driver.main(["driver.py", str(task_path)]) + + result = json.loads((tmp_path / "out" / "result.json").read_text()) + assert exit_code == 1 + assert result["status"] == "error" + assert result["error"].startswith("KeyError") + + +def test_a_failed_optional_review_is_reported_by_the_run_and_the_trace_checks(tmp_path): + from code4me_swebench import check + + workspace = tmp_path / "repo" + workspace.mkdir() + (workspace / "a.py").write_text("VALUE = 1\n") + script = [ + {"tool_calls": [_call("c1", "read_file", path="a.py")]}, + {"tool_calls": [_call("c2", "replace_text", path="a.py", old_text="VALUE = 1", + new_text="VALUE = 2")]}, + {"final_answer": "Changed VALUE to 2."}, + # No entry for the self-review call: the scripted provider raises. + ] + task_path = _task(tmp_path, workspace, script) + task = json.loads(task_path.read_text()) + task["agent_config"]["harness_options"] = {"verify_on_stop": False, "self_review": True} + task_path.write_text(json.dumps(task)) + + assert driver.main(["driver.py", str(task_path)]) == 0 + + trace = tmp_path / "out" / "trace.jsonl" + events = telemetry.read_events(trace) + run_completed = next(e for e in events if e["event_type"] == "agent.run.completed") + assert run_completed["metrics"]["failed_side_calls"] == 1 + assert run_completed["payload"]["side_call_failures"][0]["call_purpose"] == "self_review" + assert telemetry.summarize(trace)["failed_side_calls"] == 1 + assert check.check_trace(trace) == [] diff --git a/benchmarks/swebench/tests/test_leaderboard_protocol.py b/benchmarks/swebench/tests/test_leaderboard_protocol.py new file mode 100644 index 0000000..0724c03 --- /dev/null +++ b/benchmarks/swebench/tests/test_leaderboard_protocol.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import json +import math +from statistics import stdev + +import pytest +from code4me_swebench import cli, dataset, report, runner +from code4me_swebench.settings import RunSettings + + +def test_windows_are_end_exclusive_by_creation_date(): + rows = [{"instance_id": f"t-{day}", "created_at": f"2026-04-{day:02d}T10:00:00Z"} for day in (1, 14, 15, 30)] + assert dataset.in_window(rows, "2026-04-01", "2026-04-15") == ["t-1", "t-14"] + assert dataset.in_window(rows, "2026-04-15", "2026-05-01") == ["t-15", "t-30"] + assert dataset.default_revision(dataset.REBENCH_DATASET) == dataset.REBENCH_REVISION + assert dataset.default_revision(dataset.DATASET) is None + + +def test_bundled_windows_match_the_leaderboard_task_counts(): + counts = {name: len(dataset.read_subset(dataset.resolve_subset(name))) + for name in ("rebench-2026_03-0301-0515", "rebench-2026_03-0301-0401", "rebench-2026_03-0401-0515", + "rebench-2026_03-0415-0515", "rebench-2026_03-0501-0515")} + assert list(counts.values()) == [110, 46, 64, 39, 21] + + +def test_defaults_follow_the_leaderboard_protocol(): + settings = RunSettings() + assert (settings.max_iterations, settings.context_tokens) == (500, 128_000) + assert (settings.command_timeout_s, settings.max_command_timeout_s) == (300.0, 300.0) + args = cli.build_parser().parse_args( + ["run", "--run-name", "x", "--base-url", "https://openrouter.ai/api/v1", + "--model", "deepseek/deepseek-v4-flash", "--openrouter-upstream", "deepinfra"]) + routed = cli._settings(args) + assert routed.temperature == 1.0 + assert routed.extra_body["top_p"] == 1.0 and routed.extra_body["reasoning"] == {"effort": "high"} + assert cli._settings(cli.build_parser().parse_args(["run", "--run-name", "x"])).temperature is None + assert cli.repeat_names("base", 1) == ["base"] + assert cli.repeat_names("base", 3) == ["base-r1", "base-r2", "base-r3"] + + +def test_repeats_run_task_by_task_and_pull_each_image_once(tmp_path, monkeypatch): + calls, removed = [], [] + monkeypatch.setenv("TEST_KEY", "x") + monkeypatch.setattr(runner.sandbox, "ensure", lambda base_url: "model.test") + monkeypatch.setattr(runner.docker_cli, "remove_image", removed.append) + + def fake_run_instance(row, *, run_dir, run_name, settings, runtime, options): + assert options.remove_images is False # images go once, after the last attempt + calls.append((row["instance_id"], run_name)) + return {"instance_id": row["instance_id"], "agent_status": "completed"} + + monkeypatch.setattr(runner, "run_instance", fake_run_instance) + rows = [{"instance_id": "a", "image": "img-a"}, {"instance_id": "b", "image": "img-b"}] + runs = [(tmp_path / "r1", "r1"), (tmp_path / "r2", "r2")] + records = runner.run_repeated(rows, runs=runs, settings=RunSettings(api_key_env="TEST_KEY"), + runtime=None, options=runner.RunOptions(remove_images=True), workers=1) + + assert calls == [("a", "r1"), ("a", "r2"), ("b", "r1"), ("b", "r2")] + assert removed == ["img-a", "img-b"] + assert {record["run"] for record in records} == {"r1", "r2"} + + +def test_repeats_stop_after_three_outages_in_a_row(tmp_path, monkeypatch): + calls = [] + monkeypatch.setenv("TEST_KEY", "x") + monkeypatch.setattr(runner.sandbox, "ensure", lambda base_url: "model.test") + monkeypatch.setattr(runner.docker_cli, "remove_image", lambda image: None) + + def spent_key(row, *, run_dir, run_name, settings, runtime, options): + calls.append((row["instance_id"], run_name)) + return {"instance_id": row["instance_id"], "agent_status": "infra_error", "status_code": 402} + + monkeypatch.setattr(runner, "run_instance", spent_key) + rows = [{"instance_id": i, "image": f"img-{i}"} for i in ("a", "b", "c")] + runs = [(tmp_path / f"r{n}", f"r{n}") for n in (1, 2)] + runner.run_repeated(rows, runs=runs, settings=RunSettings(api_key_env="TEST_KEY"), + runtime=None, options=runner.RunOptions(), workers=1) + assert len(calls) == 3 # a/r1, a/r2, b/r1: then the run stops + + +def _run_dir(base, name, resolved: dict, *, created: dict, tokens=(1000, 900, 100), cost=0.01): + run_dir = base / name + ids = list(resolved) + (run_dir / "instances").mkdir(parents=True) + (run_dir / "manifest.json").write_text(json.dumps({"instance_ids": ids, "settings": {"model": "m"}, + "dataset": "d", "split": "s"})) + (run_dir / "dataset.json").write_text(json.dumps([{"instance_id": i, "created_at": created[i]} for i in ids])) + for instance_id, ok in resolved.items(): + directory = run_dir / "instances" / instance_id + directory.mkdir() + (directory / "prediction.json").write_text("{}") + (directory / "result.json").write_text(json.dumps({"status": "completed"})) + (directory / "eval.json").write_text(json.dumps({"resolved": ok, "status": "resolved" if ok else "failed"})) + prompt, cached, out = tokens + (directory / "telemetry.json").write_text(json.dumps({ + "model_calls": 10, "prompt_tokens": prompt, "cached_prompt_tokens": cached, + "completion_tokens": out, "cost_usd": cost})) + return run_dir + + +def test_aggregate_uses_the_leaderboard_formulas(tmp_path): + created = {"a": "2026-03-10", "b": "2026-04-20", "c": "2026-05-05", "d": "2026-05-10"} + outcomes = [ + {"a": True, "b": False, "c": True, "d": False}, + {"a": True, "b": True, "c": False, "d": False}, + {"a": False, "b": False, "c": True, "d": False}, + ] + dirs = [_run_dir(tmp_path, f"x-r{n}", o, created=created) for n, o in enumerate(outcomes, start=1)] + + result = report.aggregate_runs(dirs, windows=[("2026-05-01", "2026-05-15")]) + + rates = [50.0, 50.0, 25.0] + overall = result["overall"] + assert overall["resolved_rate"] == round(sum(rates) / 3, 2) + assert overall["sem"] == round(stdev(rates) / math.sqrt(3), 2) + assert overall["pass_at_3"] == 75.0 # a, b, c solved at least once; d never + assert overall["tokens_per_problem"] == 1100 and overall["cached_share"] == 90.0 + assert overall["cost_per_problem"] == 0.01 + may = result["windows"]["2026-05-01..2026-05-15"] + assert (may["tasks"], may["per_run_rates"]) == (2, [50.0, 0.0, 50.0]) + + +def test_aggregate_compares_only_the_same_task_window_with_a_reference(tmp_path, monkeypatch): + refs = {"references": {"ref": {"label": "R", "windows": { + "2026-05-01..2026-05-15": {"tasks": 2, "resolved_rate": 40.0, "sem": 3.0}, + "2026-04-01..2026-05-15": {"tasks": 64, "resolved_rate": 40.0, "sem": 2.0}}}}} + path = tmp_path / "references.json" + path.write_text(json.dumps(refs)) + monkeypatch.setattr(report, "REFERENCES", path) + created = {"c": "2026-05-05", "d": "2026-05-10"} + dirs = [_run_dir(tmp_path, f"y-r{n}", {"c": n == 1, "d": False}, created=created) for n in (1, 2)] + + result = report.aggregate_runs(dirs, windows=[("2026-05-01", "2026-05-15"), ("2026-04-01", "2026-05-15")], + reference="ref") + + may = result["windows"]["2026-05-01..2026-05-15"] + assert may["reference"]["difference"] == round(may["resolved_rate"] - 40.0, 2) + assert "reference" not in result["windows"]["2026-04-01..2026-05-15"] # 2 tasks here, 64 there + with pytest.raises(SystemExit): + report.aggregate_runs(dirs, reference="missing") diff --git a/benchmarks/swebench/tests/test_openrouter.py b/benchmarks/swebench/tests/test_openrouter.py new file mode 100644 index 0000000..08d1d68 --- /dev/null +++ b/benchmarks/swebench/tests/test_openrouter.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import json + +import pytest +from code4me_swebench import baseline_mini, check, cli, telemetry +from code4me_swebench.settings import ( + RunSettings, + agent_config, + openrouter_routing, + pinned_upstreams, +) + +OPENROUTER = "https://openrouter.ai/api/v1" + + +def _settings(*flags): + args = cli.build_parser().parse_args( + ["run", "--run-name", "x", "--base-url", OPENROUTER, "--model", "deepseek/deepseek-v4.1-flash", *flags]) + return cli._settings(args) + + +def test_openrouter_runs_must_pin_one_upstream_and_both_agents_send_it(): + with pytest.raises(SystemExit, match="openrouter-upstream"): + _settings() + with pytest.raises(SystemExit, match="openrouter-upstream"): + _settings("--extra-body", '{"provider": {"order": ["deepseek"]}}') # fallbacks still allowed + settings = _settings("--openrouter-upstream", "deepseek") + assert pinned_upstreams(settings.extra_body) == ["deepseek"] + assert settings.extra_body["provider"]["allow_fallbacks"] is False + assert settings.extra_body["reasoning"] == {"effort": "high"} # explicit, overridable + assert _settings("--openrouter-upstream", "deepseek", + "--extra-body", '{"reasoning": {"effort": "max"}}').extra_body["reasoning"] == {"effort": "max"} + assert agent_config(settings, session_id="s")["adapter"]["provider"]["extra_body"] == settings.extra_body + assert settings.as_dict()["extra_body"] == settings.extra_body + overlay = baseline_mini.overlay_config(settings) + assert overlay["model"]["model_kwargs"]["extra_body"] == settings.extra_body + assert baseline_mini.model_class_args(settings) == [] # stock litellm: no OpenCode header + # Runs without extra fields keep their manifest and config shape. + plain = RunSettings() + assert "extra_body" not in plain.as_dict() + assert "extra_body" not in agent_config(plain, session_id="s")["adapter"]["provider"] + assert baseline_mini.model_class_args(plain) == ["--model-class", "opencode_model.OpenCodeModel"] + + +def _completed(index: int, provider: str | None, cost: float) -> dict: + response = {"usage": {"prompt_tokens": 1, "completion_tokens": 1, "cost": cost}} + if provider: + response["provider"] = provider + return {"event_type": "agent.model.completed", "timestamp": f"2026-10-14T00:00:0{index}Z", + "payload": {"usage_estimated": True}, "raw_payload": {"response": response}} + + +def test_calls_from_unpinned_upstreams_fail_the_check_and_spend_is_summed(tmp_path): + trace = tmp_path / "trace.jsonl" + events = [_completed(1, "DeepSeek", 0.002), _completed(2, "Novita", 0.001), _completed(3, None, 0.0)] + trace.write_text("".join(json.dumps(event) + "\n" for event in events)) + + problems = check.check_trace(trace, upstreams=openrouter_routing(["deepseek"])["provider"]["only"]) + assert "model calls served by unpinned upstreams: {'Novita': 1}" in problems + assert "1 model calls without the serving upstream in the response" in problems + assert not [p for p in check.check_trace(trace) if "upstream" in p] # nothing pinned, nothing checked + summary = telemetry.summarize(trace) + assert summary["upstreams"] == {"DeepSeek": 1, "Novita": 1} + assert summary["cost_usd"] == 0.003 + + +def test_parsed_test_runs_are_counted(tmp_path): + trace = tmp_path / "trace.jsonl" + run = {"event_type": "agent.tool.completed", "payload": { + "tool_name": "run_command", "argv": ["python", "tests/runtests.py"], "status": "ok", + "test_framework": "unittest", "tests_passed": 79, "tests_failed": 0}} + other = {"event_type": "agent.tool.completed", "payload": { + "tool_name": "run_command", "argv": ["git", "status"], "status": "ok"}} + trace.write_text(json.dumps(run) + "\n" + json.dumps(other) + "\n") + assert telemetry.summarize(trace)["test_runs"] == 1 diff --git a/benchmarks/swebench/tests/test_outages.py b/benchmarks/swebench/tests/test_outages.py new file mode 100644 index 0000000..d4f9338 --- /dev/null +++ b/benchmarks/swebench/tests/test_outages.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import json + +from code4me_swebench import runner + + +def _task(run, iid, *, result, events=(), telemetry=None): + directory = run / "instances" / iid + directory.mkdir(parents=True) + (directory / "result.json").write_text(json.dumps(result)) + (directory / "trace.jsonl").write_text("".join(json.dumps(e) + "\n" for e in events)) + (directory / "prediction.json").write_text(json.dumps({"instance_id": iid, "model_patch": ""})) + (directory / "eval.json").write_text(json.dumps({"resolved": False, "status": "empty_patch"})) + if telemetry is not None: + (directory / "telemetry.json").write_text(json.dumps(telemetry)) + return directory + + +def test_provider_outage_is_told_apart_from_an_agent_failure(tmp_path): + rate_limited = _task(tmp_path, "a__a-1", result={"run_status": "failed", "stop_reason": "error", + "final_response": "HTTP 429 after 3 attempts"}, + events=[{"event_type": "agent.adapter.loop_failed", + "payload": {"failure_reason": "provider_request_failed", + "error_type": "ProviderRequestFailed"}, + "metrics": {"status_code": 429, "attempts": 3}}]) + finished = _task(tmp_path, "b__b-2", result={"run_status": "completed", "stop_reason": "end_turn"}) + gave_up = _task(tmp_path, "c__c-3", result={"run_status": "failed", "stop_reason": "max_iterations"}) + + assert runner.provider_outage(rate_limited)["status_code"] == 429 + assert runner.provider_outage(finished) is None + assert runner.provider_outage(gave_up) is None + + +def test_requeue_archives_outage_attempts_and_drops_their_predictions(tmp_path): + run = tmp_path / "run" + ids = ["a__a-1", "b__b-2"] + run.mkdir() + (run / "manifest.json").write_text(json.dumps({"instance_ids": ids, "settings": {}})) + _task(run, "a__a-1", result={"run_status": "failed", "stop_reason": "error"}) + _task(run, "b__b-2", result={"run_status": "completed", "stop_reason": "end_turn"}) + stale_report = run / "logs" / "run_evaluation" / "run" / "m" / "a__a-1" + stale_report.mkdir(parents=True) + + assert runner.requeue_outages(run) == ["a__a-1"] + + a = run / "instances" / "a__a-1" + assert not (a / "prediction.json").exists() and (a / "attempts" / "1" / "trace.jsonl").exists() + assert not stale_report.exists() + assert (run / "instances" / "b__b-2" / "prediction.json").exists() + assert [json.loads(line)["instance_id"] for line in (run / "predictions.jsonl").read_text().splitlines()] == ["b__b-2"] + + +def test_requeue_uses_mini_exit_statuses_for_baseline_runs(tmp_path): + run = tmp_path / "mini" + run.mkdir() + (run / "manifest.json").write_text(json.dumps({"instance_ids": ["a__a-1", "b__b-2"], "baseline": {}})) + _task(run, "a__a-1", result={"status": "completed"}, telemetry={"exit_status": "RateLimitError"}) + _task(run, "b__b-2", result={"status": "completed"}, telemetry={"exit_status": "Submitted"}) + + assert runner.requeue_outages(run) == ["a__a-1"] + + +def test_a_rejected_request_is_the_agents_failure_not_an_outage(tmp_path): + bad_request = _task(tmp_path, "d__d-4", result={"run_status": "failed", "stop_reason": "error"}, + events=[{"event_type": "agent.adapter.loop_failed", + "payload": {"failure_reason": "provider_request_failed"}, + "metrics": {"status_code": 400}}]) + assert runner.provider_outage(bad_request) is None + + +def test_requeue_finalizes_a_mislabelled_rejection_for_grading(tmp_path): + run = tmp_path / "run" + run.mkdir() + (run / "manifest.json").write_text(json.dumps({"instance_ids": ["d__d-4"], "settings": {"model": "m"}})) + directory = _task(run, "d__d-4", result={"run_status": "failed", "stop_reason": "error"}, + events=[{"event_type": "agent.adapter.loop_failed", + "payload": {"failure_reason": "provider_request_failed"}, + "metrics": {"status_code": 400}}]) + (directory / "prediction.json").unlink() + (directory / "eval.json").unlink() + (directory / "patch.diff").write_text("diff --git a/x b/x\n") + (directory / "infra_error.json").write_text("{}") + + assert runner.requeue_outages(run) == [] + assert json.loads((directory / "prediction.json").read_text())["model_patch"].startswith("diff --git") + assert not (directory / "infra_error.json").exists() + + +def test_an_exhausted_or_refused_key_is_an_outage_not_a_failed_task(tmp_path): + # An OpenRouter key at its credit cap answers 402 (or 403) to every call; a + # pinned upstream that is unavailable gives 404. None of it is the agent's doing. + for index, status in enumerate((401, 402, 403, 404)): + refused = _task(tmp_path, f"k__k-{index}", result={"run_status": "failed", "stop_reason": "error"}, + events=[{"event_type": "agent.adapter.loop_failed", + "payload": {"failure_reason": "provider_request_failed", + "error_type": "ProviderRequestFailed"}, + "metrics": {"status_code": status, "attempts": 1}}]) + assert runner.provider_outage(refused)["status_code"] == status diff --git a/benchmarks/swebench/tests/test_rebench_and_baseline.py b/benchmarks/swebench/tests/test_rebench_and_baseline.py new file mode 100644 index 0000000..36c31d4 --- /dev/null +++ b/benchmarks/swebench/tests/test_rebench_and_baseline.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import sys + +from code4me_swebench import baseline_mini, dataset, evaluate +from code4me_swebench.settings import RunSettings + + +def test_rebench_rows_get_their_image_and_grader(): + row = dataset.normalize_row({"instance_id": "a__b-1", "image_name": "swerebench/x:latest", + "docker_image": "swerebench/y:latest"}) + assert row["image"] == "swerebench/x:latest" + verified = dataset.normalize_row({"instance_id": "c__d-2", "image": "swebench/z:latest"}) + assert verified["image"] == "swebench/z:latest" + assert dataset.default_grader(dataset.REBENCH_DATASET) == "rebench" + assert dataset.default_grader(dataset.DATASET) == "swebench" + + +def test_newest_takes_the_latest_created_tasks(): + rows = [{"instance_id": f"r__r-{n}", "created_at": f"2026-0{n}-01"} for n in range(1, 8)] + assert dataset.newest(rows, 3) == ["r__r-5", "r__r-6", "r__r-7"] + + +def test_bundled_rebench_subset_is_fifty_ids(): + assert len(dataset.read_subset(dataset.resolve_subset("rebench-2026_03-newest-50"))) == 50 + + +def test_swebench_grader_uses_this_interpreter(): + assert evaluate.grader_command("swebench")[0] == sys.executable + + +def test_mini_overlay_only_changes_endpoint_platform_and_cost_tracking(): + overlay = baseline_mini.overlay_config(RunSettings(model="deepseek-v4-flash")) + assert set(overlay) == {"model", "environment"} + assert overlay["model"]["model_name"] == "openai/deepseek-v4-flash" + assert overlay["model"]["model_kwargs"]["api_base"] == "https://opencode.ai/zen/go/v1" + assert overlay["model"]["cost_tracking"] == "ignore_errors" + # No network in mini's task containers: it calls the model from the host. + assert overlay["environment"]["run_args"] == ["--rm", "--platform", "linux/amd64", "--network", "none"] + assert "step_limit" not in str(overlay) diff --git a/benchmarks/swebench/tests/test_sandbox.py b/benchmarks/swebench/tests/test_sandbox.py new file mode 100644 index 0000000..be50a50 --- /dev/null +++ b/benchmarks/swebench/tests/test_sandbox.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import asyncio + +from code4me_swebench import egress_proxy, sandbox +from code4me_swebench.settings import RunSettings + +HOSTS = egress_proxy.parse_hosts("opencode.ai") +PORTS = frozenset({443}) + + +def test_only_the_model_host_and_its_subdomains_on_443_are_allowed(): + def allowed(host, port=443): + return egress_proxy.is_allowed(host, port, hosts=HOSTS, ports=PORTS) + + assert allowed("opencode.ai") and allowed("api.opencode.ai") and allowed("OpenCode.AI.") + assert not allowed("github.com") and not allowed("pypi.org") and not allowed("files.pythonhosted.org") + assert not allowed("evil-opencode.ai") and not allowed("opencode.ai.evil.com") + assert not allowed("opencode.ai", 80) and not allowed("") + + +def test_connect_lines_are_parsed_and_plain_requests_are_not(): + assert egress_proxy.parse_connect("CONNECT opencode.ai:443 HTTP/1.1") == ("opencode.ai", 443) + assert egress_proxy.parse_connect("CONNECT [::1]:443 HTTP/1.1") == ("::1", 443) + assert egress_proxy.parse_connect("GET http://pypi.org/simple/ HTTP/1.1") is None + assert egress_proxy.parse_connect("CONNECT github.com HTTP/1.1") is None + + +def test_the_proxy_tunnels_to_allowed_targets_and_refuses_the_rest(): + async def scenario(): + async def echo(reader, writer): + writer.write(await reader.read(100)) + await writer.drain() + writer.close() + + upstream = await asyncio.start_server(echo, "127.0.0.1", 0) + up_port = upstream.sockets[0].getsockname()[1] + proxy = await egress_proxy.serve("127.0.0.1", 0, hosts=frozenset({"127.0.0.1"}), + ports=frozenset({up_port})) + proxy_port = proxy.sockets[0].getsockname()[1] + + async def connect(target: str) -> bytes: + reader, writer = await asyncio.open_connection("127.0.0.1", proxy_port) + writer.write(f"CONNECT {target} HTTP/1.1\r\nHost: {target}\r\n\r\n".encode()) + await writer.drain() + status = await reader.readuntil(b"\r\n\r\n") + if b" 200 " in status: + writer.write(b"ping") + await writer.drain() + status += await reader.read(100) + writer.close() + return status + + allowed = await connect(f"127.0.0.1:{up_port}") + denied_host = await connect(f"github.com:{up_port}") + denied_port = await connect(f"127.0.0.1:{up_port + 1}") + proxy.close() + upstream.close() + return allowed, denied_host, denied_port + + allowed, denied_host, denied_port = asyncio.run(scenario()) + assert b" 200 " in allowed and allowed.endswith(b"ping") + assert b" 403 " in denied_host and b" 403 " in denied_port + + +def test_task_containers_get_the_proxy_in_both_spellings_and_the_policy_is_recorded(): + env = dict(item.split("=", 1) for item in sandbox.proxy_env()) + assert env["HTTPS_PROXY"] == env["https_proxy"] == "http://code4me-swebench-proxy:3128" + assert "localhost" in env["NO_PROXY"] + assert sandbox.model_host("https://opencode.ai/zen/go/v1") == "opencode.ai" + assert RunSettings().as_dict()["network"] == "model-api-only" diff --git a/benchmarks/swebench/tests/test_selection_and_report.py b/benchmarks/swebench/tests/test_selection_and_report.py new file mode 100644 index 0000000..8acd099 --- /dev/null +++ b/benchmarks/swebench/tests/test_selection_and_report.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import json +from collections import Counter + +import pytest +from code4me_swebench import dataset, report +from code4me_swebench.prompt import task_prompt + + +def _rows() -> list[dict]: + rows = [] + for repo, difficulty, count in [ + ("django/django", "<15 min fix", 40), ("django/django", "15 min - 1 hour", 30), + ("sympy/sympy", "15 min - 1 hour", 20), ("psf/requests", "1-4 hours", 7), + ("pallets/flask", "<15 min fix", 3), + ]: + for index in range(count): + rows.append({"instance_id": f"{repo.split('/')[1]}-{difficulty[:2]}-{index}", + "repo": repo, "difficulty": difficulty}) + return rows + + +def test_stratified_sample_is_exact_proportional_and_deterministic(): + rows = _rows() + first = dataset.stratified_sample(rows, 20, seed=3) + assert len(first) == len(set(first)) == 20 + assert first == dataset.stratified_sample(rows, 20, seed=3) + assert first != dataset.stratified_sample(rows, 20, seed=4) + by_repo = Counter(iid.split("-")[0] for iid in first) + # Quotas of 20: django 8 + 6, sympy 4, requests 1.4, flask 0.6. The last + # seat goes to the largest remainder (flask 0.6 beats requests 0.4). + assert by_repo == {"django": 14, "sympy": 4, "requests": 1, "flask": 1} + + +def test_read_subset_ignores_comments_and_rejects_duplicates(tmp_path): + path = tmp_path / "s.txt" + path.write_text("# header\na__a-1\n\nb__b-2 # note\n") + assert dataset.read_subset(path) == ["a__a-1", "b__b-2"] + path.write_text("a__a-1\na__a-1\n") + with pytest.raises(ValueError): + dataset.read_subset(path) + + +def test_select_rows_keeps_order_and_reports_missing_ids(): + rows = [{"instance_id": "x"}, {"instance_id": "y"}] + assert [row["instance_id"] for row in dataset.select_rows(rows, ["y", "x"])] == ["y", "x"] + with pytest.raises(KeyError): + dataset.select_rows(rows, ["z"]) + + +def test_bundled_subsets_have_fifty_distinct_ids(): + for name in ("verified-mini", "verified-stratified-50"): + ids = dataset.read_subset(dataset.resolve_subset(name)) + assert len(ids) == 50 and all("__" in iid for iid in ids) + + +def test_wilson_interval_matches_reference_values(): + low, high = report.wilson_interval(25, 50) + assert (round(low, 3), round(high, 3)) == (0.366, 0.634) + assert report.wilson_interval(0, 0) == (0.0, 0.0) + + +def test_report_counts_ungraded_tasks_as_unresolved(tmp_path): + run = tmp_path / "run" + ids = ["a__a-1", "b__b-2", "c__c-3"] + (run / "instances").mkdir(parents=True) + (run / "manifest.json").write_text(json.dumps( + {"instance_ids": ids, "settings": {"model": "m"}, "subset": "s"})) + for iid, resolved in (("a__a-1", True), ("b__b-2", False)): + directory = run / "instances" / iid + directory.mkdir() + (directory / "prediction.json").write_text("{}") + (directory / "result.json").write_text(json.dumps({"status": "completed"})) + (directory / "eval.json").write_text(json.dumps( + {"resolved": resolved, "status": "resolved" if resolved else "unresolved"})) + (directory / "telemetry.json").write_text(json.dumps( + {"model_calls": 4, "tool_calls": {"read_file": 2}})) + (directory / "patch.diff").write_text("--- a/x\n+++ b/x\n-old\n+new\n") + + data = report.write_report(run) + + assert (data["instances"], data["graded"], data["resolved"]) == (3, 2, 1) + assert data["resolve_rate"] == round(1 / 3, 4) + assert data["agent_status"] == {"completed": 2, "not_run": 1} + assert data["model_calls"] == {"total": 8, "median": 4} + table = (run / "instances.csv").read_text().splitlines() + assert len(table) == 4 and "patch_lines" in table[0] + + +def test_task_prompt_carries_the_issue_but_no_hints(): + text = task_prompt(" The widget crashes.\n", workspace="/testbed") + assert "\nThe widget crashes.\n" in text + assert "/testbed" in text + assert "hint" not in text.lower() + + +def test_mcnemar_exact_p_values(): + assert report.mcnemar_exact_p(0, 0) == 1.0 + assert report.mcnemar_exact_p(3, 3) == 1.0 + # 8 vs 1 discordant pairs: 2 * P(X <= 1), X ~ Bin(9, 0.5) = 2 * 10/512. + assert round(report.mcnemar_exact_p(8, 1), 4) == round(20 / 512, 4) + + +def test_no_self_review_flag_is_explicit_in_the_settings(): + from code4me_swebench import cli + + def settings(*flags): + return cli._settings(cli.build_parser().parse_args(["run", "--run-name", "x", *flags])) + + assert settings().harness_options == {} + assert settings("--no-self-review").harness_options == {"self_review": False} + assert settings("--no-self-review").as_dict()["harness_options"] == {"self_review": False} diff --git a/docker-compose.dev-arm.yaml b/docker-compose.dev-arm.yaml new file mode 100644 index 0000000..ffcd0cd --- /dev/null +++ b/docker-compose.dev-arm.yaml @@ -0,0 +1,262 @@ +# Local dev stack for machines without an NVIDIA GPU (Apple Silicon / aarch64, +# or any x86_64 box without CUDA). Mirrors docker-compose.yml but: +# - builds backend/celery-worker from Dockerfile.cpu instead of Dockerfile +# - drops GPU device reservations, NVIDIA_*/CUDA_* env vars, and the NVML +# library bind-mounts +# - defaults PRELOAD_MODELS=false, since loading a multi-GB transformer +# model on CPU is impractical for local testing and unnecessary for +# exercising the agent subsystem (which calls an external OpenAI-compatible +# upstream such as OpenRouter, not a locally loaded model) +# - serves the classic completion/chat endpoints from the seeded +# provider-backed rows (OpenRouter, OPENROUTER_API_KEY in .env); a local +# HuggingFace model loads only for a row explicitly configured as local. +# CLASSIC_MODELS_ENABLED=false switches the classic endpoints off. +# +# Usage: docker compose -f docker-compose.dev-arm.yaml up -d --build + +services: + db: + image: pgvector/pgvector:pg16 + container_name: postgres + ports: + - ${DB_PORT:-2345}:${DB_PORT:-2345} + expose: + - ${DB_PORT:-2345} + volumes: + - .:/app + - ./src/database/init.sql:/docker-entrypoint-initdb.d/init.sql + # Persisted data directory: without this, `docker compose down` removes + # the container and the whole database with it. Reset explicitly with + # `docker compose ... down -v` when a fresh database is wanted. + - pgdata:/var/lib/postgresql/data + environment: + - POSTGRES_PASSWORD=${DB_PASSWORD:-postgres} + - POSTGRES_USER=${DB_USER:-postgres} + - POSTGRES_DB=${DB_NAME:-code4meV2} + - PGPORT=${DB_PORT:-2345} + command: ["postgres", "-p", "${DB_PORT:-2345}"] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -p ${DB_PORT:-2345}"] + start_period: 0s + start_interval: 0s + interval: 60s + timeout: 5s + retries: 5 + + redis: + image: redis:7.0 + container_name: redis + ports: + - ${REDIS_PORT:-6379}:${REDIS_PORT:-6379} + expose: + - ${REDIS_PORT:-6379} + volumes: + - .:/app + - ./redis.conf:/usr/local/etc/redis/redis.conf + command: ["redis-server", "/usr/local/etc/redis/redis.conf", "--port", "${REDIS_PORT:-6379}"] + healthcheck: + test: ["CMD", "redis-cli", "-p", "${REDIS_PORT:-6379}", "ping"] + start_period: 0s + start_interval: 0s + interval: 60s + timeout: 5s + retries: 3 + + redis-celery: + image: redis:7.0 + container_name: redis-celery + ports: + - ${CELERY_BROKER_PORT:-6380}:${CELERY_BROKER_PORT:-6380} + expose: + - ${CELERY_BROKER_PORT:-6380} + command: ["redis-server", "--port", "${CELERY_BROKER_PORT:-6380}"] + volumes: + - .:/app + healthcheck: + test: ["CMD", "redis-cli", "-p", "${CELERY_BROKER_PORT}", "ping"] + start_period: 0s + start_interval: 0s + interval: 30s + timeout: 5s + retries: 3 + + celery-worker: + build: + context: . + dockerfile: Dockerfile.cpu + container_name: celery-worker + env_file: + - ./.env + command: [ + "/bin/bash", + "-c", + "source activate myenv && celery -A celery_app.celery_app worker --loglevel=info -E -Q llm,db --concurrency=1 --prefetch-multiplier=1", + ] + depends_on: + redis-celery: + condition: service_healthy + db: + condition: service_healthy + environment: + - REDIS_HOST=redis + - REDIS_PORT=${REDIS_PORT:-6379} + - CELERY_BROKER_HOST=redis-celery + - CELERY_BROKER_PORT=${CELERY_BROKER_PORT:-6380} + - DB_HOST=db + - DB_PORT=${DB_PORT:-2345} + - DB_NAME=${DB_NAME:-code4meV2} + - DB_USER=${DB_USER:-postgres} + - DB_PASSWORD=${DB_PASSWORD:-postgres} + - TEST_DATABASE_URL=postgresql://postgres:postgres@test_db:5432/test_db + - PYTHONPATH=/app/src + - PYTHONMULTIPROCESSING_START_METHOD=spawn + - TOKENIZERS_PARALLELISM=false + - OMP_NUM_THREADS=1 + - HF_TOKEN=${HF_TOKEN:-} + - HF_HOME=/app/hf_cache + - HF_HUB_CACHE=/app/hf_cache + - TRANSFORMERS_CACHE=/app/hf_cache + - MODEL_CACHE_DIR=/app/hf_cache + - CELERY_WORKER=true + - PRELOAD_MODELS=false + - CLASSIC_MODELS_ENABLED=${CLASSIC_MODELS_ENABLED:-true} + volumes: + - .:/app + - ${DATA_DIR:-./data}/hf:/app/hf_cache + healthcheck: + test: + [ + "CMD", + "bash", + "-c", + "source activate myenv && celery -A celery_app.celery_app status --timeout=5", + ] + start_period: 30s + interval: 60s + timeout: 20s + retries: 3 + + backend: + build: + context: . + dockerfile: Dockerfile.cpu + container_name: backend + env_file: + - ./.env + command: ["/bin/bash", "-c", "source activate myenv && python /app/src/main.py"] + expose: + - ${SERVER_PORT:-8008} + ports: + - "${SERVER_PORT:-8008}:8008" + volumes: + - .:/app + - ${DATA_DIR:-./data}/hf:/app/hf_cache + environment: + - SERVER_PORT=${SERVER_PORT:-8008} + - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS:-} + - REDIS_HOST=redis + - REDIS_PORT=${REDIS_PORT:-6379} + - CELERY_BROKER_HOST=redis-celery + - CELERY_BROKER_PORT=${CELERY_BROKER_PORT:-6380} + - DB_HOST=db + - DB_PORT=${DB_PORT:-2345} + - DB_NAME=${DB_NAME:-code4meV2} + - DB_USER=${DB_USER:-postgres} + - DB_PASSWORD=${DB_PASSWORD:-postgres} + - WEBSITE_HOST=website + - WEBSITE_PORT=${WEBSITE_PORT:-3000} + - PYTHONPATH=/app/src + - HF_TOKEN=${HF_TOKEN:-} + - HF_HOME=/app/hf_cache + - HF_HUB_CACHE=/app/hf_cache + - TRANSFORMERS_CACHE=/app/hf_cache + - MODEL_CACHE_DIR=/app/hf_cache + - PRELOAD_MODELS=false + - CLASSIC_MODELS_ENABLED=${CLASSIC_MODELS_ENABLED:-true} + # Local dev stack: restart the API process when Python sources change so + # the running server never serves stale in-memory code. Set to 0 to opt out. + - CODE4ME_DEV_RELOAD=${CODE4ME_DEV_RELOAD:-1} + # Agent subsystem (ACP) upstream provider config, see .env.example + - AGENT_UPSTREAM_BASE_URL=${AGENT_UPSTREAM_BASE_URL:-http://host.docker.internal:11434/v1} + - AGENT_UPSTREAM_API_KEY=${AGENT_UPSTREAM_API_KEY:-} + - OLLAMA_API_KEY=${OLLAMA_API_KEY:-} + - OPENAI_API_KEY=${OPENAI_API_KEY:-} + - GROQ_API_KEY=${GROQ_API_KEY:-} + - OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-} + - AGENT_REASONING_EFFORT=${AGENT_REASONING_EFFORT:-} + # Lets AGENT_UPSTREAM_BASE_URL=http://host.docker.internal:11434/v1 reach an + # Ollama instance running on the host from inside this container. + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + celery-worker: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "--spider", "http://127.0.0.1:8008/api/ping"] + start_period: 0s + interval: 60s + timeout: 5s + retries: 3 + + website: + image: node:18 + container_name: website + working_dir: /website + command: bash -c "npm install && npm start -- --port ${WEBSITE_PORT:-3000}" + expose: + - ${WEBSITE_PORT:-3000} + ports: + - "${WEBSITE_PORT:-3000}:${WEBSITE_PORT:-3000}" + volumes: + - ./src/website:/website + - website_node_modules:/website/node_modules + depends_on: + backend: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "--spider", "http://localhost:${WEBSITE_PORT:-3000}"] + start_period: 0s + interval: 60s + timeout: 5s + retries: 3 + + nginx: + image: nginx:1.25 + container_name: nginx + ports: + - "8000:8000" + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf + depends_on: + website: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000"] + start_period: 0s + interval: 60s + timeout: 5s + retries: 3 + + test_db: + profiles: ["test"] + image: pgvector/pgvector:pg16 + container_name: postgres_test + ports: + - "5433:5432" + environment: + - POSTGRES_PASSWORD=postgres + - POSTGRES_USER=postgres + - POSTGRES_DB=test_db + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d test_db"] + interval: 2s + timeout: 5s + retries: 10 + +volumes: + website_node_modules: + pgdata: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..dfa2934 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,267 @@ +# Production stack for the Code4Me research platform (production-readiness +# review P0-2: B-01, B-03, B-04, B-05, B-06, B-07, B-08, B-11). +# +# docker compose --env-file .env -f docker-compose.prod.yml config # check +# docker compose --env-file .env -f docker-compose.prod.yml up -d --build +# +# Differences from docker-compose.yml (the GPU/dev stack): +# - CPU images (Dockerfile.cpu); no NVIDIA reservations, no Hugging Face login, +# PRELOAD_MODELS=false. Classic completion/chat rows are provider-backed. +# - The application reads `.env` (env_file) and the compose file asserts the +# secrets that must exist (`${VAR:?...}`), so a missing +# BOOTSTRAP_SIGNING_SECRET fails at `up`, not at the first participant. +# - restart policies, bounded json-file logs, health checks that touch the +# database and Redis, and a one-shot `migrate` job the backend waits for. +# - No source bind mounts: the image carries the code that was built. +# - Only nginx (the `website` service) is published. Postgres is bound to +# 127.0.0.1 for backups/psql; Redis and the backend are internal. +# - Redis persists (AOF) and requires a password; Postgres is dumped nightly by +# the `backup` service (scripts/ops/backup-postgres.sh). +# - The website is a static build served by nginx, which also proxies /api/. +# +# TLS terminates in front of `website` (see docs/research-platform/ +# PRODUCTION_DEPLOYMENT.md); PUBLIC_ORIGIN is the https:// origin participants use. + +x-logging: &default-logging + driver: json-file + options: + max-size: "50m" + max-file: "5" + +x-backend-environment: &backend-environment + SERVER_HOST: 0.0.0.0 + SERVER_PORT: "8008" + DB_HOST: db + DB_PORT: "5432" + DB_NAME: ${DB_NAME:?DB_NAME is required} + DB_USER: ${DB_USER:?DB_USER is required} + DB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required} + REDIS_HOST: redis + REDIS_PORT: "6379" + REDIS_PASSWORD: ${REDIS_PASSWORD:?REDIS_PASSWORD is required (redis requirepass)} + CELERY_BROKER_HOST: redis-celery + CELERY_BROKER_PORT: "6379" + PYTHONPATH: /app/src + PRELOAD_MODELS: "false" + CLASSIC_MODELS_ENABLED: ${CLASSIC_MODELS_ENABLED:-true} + HF_HOME: /app/hf_cache + HF_HUB_CACHE: /app/hf_cache + TRANSFORMERS_CACHE: /app/hf_cache + MODEL_CACHE_DIR: /app/hf_cache + BOOTSTRAP_SIGNING_SECRET: ${BOOTSTRAP_SIGNING_SECRET:?BOOTSTRAP_SIGNING_SECRET is required (research bootstrap/capability HMAC)} + # Participants reach the API through the website/nginx container only, so + # its X-Forwarded-For is trusted and rate limits key on the real client. + FORWARDED_ALLOW_IPS: "*" + COOKIE_SECURE: "true" + CORS_ALLOWED_ORIGINS: ${PUBLIC_ORIGIN:?PUBLIC_ORIGIN is required (https origin participants use)} + WEBSITE_HOST: website + WEBSITE_PORT: "80" + +services: + db: + image: pgvector/pgvector:pg16 + restart: unless-stopped + logging: *default-logging + shm_size: 256m + ports: + - "127.0.0.1:${DB_HOST_PORT:-5432}:5432" + volumes: + - ./src/database/init.sql:/docker-entrypoint-initdb.d/init.sql:ro + - postgres_data:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required} + POSTGRES_USER: ${DB_USER:?DB_USER is required} + POSTGRES_DB: ${DB_NAME:?DB_NAME is required} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 20s + + redis: + image: redis:7.2 + restart: unless-stopped + logging: *default-logging + command: + - redis-server + - /usr/local/etc/redis/redis.conf + - --requirepass + - ${REDIS_PASSWORD:?REDIS_PASSWORD is required} + environment: + REDISCLI_AUTH: ${REDIS_PASSWORD:?REDIS_PASSWORD is required} + volumes: + - ./redis.prod.conf:/usr/local/etc/redis/redis.conf:ro + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 30s + timeout: 5s + retries: 5 + + redis-celery: + image: redis:7.2 + restart: unless-stopped + logging: *default-logging + command: ["redis-server", "--appendonly", "yes", "--dir", "/data"] + volumes: + - redis_celery_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 30s + timeout: 5s + retries: 5 + + # One-shot schema job: brings the database to the code's Alembic head (and + # initializes a fresh database) before the API starts. A failure stops the + # deployment here; the previous backend, if any, keeps serving. + migrate: + build: + context: . + dockerfile: Dockerfile.cpu + image: code4me2-server-backend:prod + restart: "no" + logging: *default-logging + env_file: + - ./.env + environment: + <<: *backend-environment + command: + - /bin/bash + - -c + - source activate myenv && python src/database/migration/migration_manager.py migrate + depends_on: + db: + condition: service_healthy + + celery-worker: + build: + context: . + dockerfile: Dockerfile.cpu + image: code4me2-server-backend:prod + restart: unless-stopped + logging: *default-logging + env_file: + - ./.env + environment: + <<: *backend-environment + CELERY_WORKER: "true" + PYTHONMULTIPROCESSING_START_METHOD: spawn + TOKENIZERS_PARALLELISM: "false" + OMP_NUM_THREADS: "1" + command: + - /bin/bash + - -c + - source activate myenv && celery -A celery_app.celery_app worker --loglevel=info -E -Q llm,db --concurrency=1 --prefetch-multiplier=1 + volumes: + - hf_cache:/app/hf_cache + depends_on: + migrate: + condition: service_completed_successfully + redis-celery: + condition: service_healthy + db: + condition: service_healthy + healthcheck: + test: ["CMD", "bash", "-c", "source activate myenv && celery -A celery_app.celery_app status --timeout=5"] + interval: 60s + timeout: 20s + retries: 3 + start_period: 60s + deploy: + resources: + limits: + memory: 2g + + backend: + build: + context: . + dockerfile: Dockerfile.cpu + image: code4me2-server-backend:prod + restart: unless-stopped + logging: *default-logging + env_file: + - ./.env + environment: + <<: *backend-environment + CODE4ME_WEB_WORKERS: ${CODE4ME_WEB_WORKERS:-2} + command: ["/bin/bash", "-c", "source activate myenv && python /app/src/main.py"] + expose: + - "8008" + volumes: + - hf_cache:/app/hf_cache + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + migrate: + condition: service_completed_successfully + db: + condition: service_healthy + redis: + condition: service_healthy + # The API must come up even if the classic worker is unhealthy: the + # research plane does not depend on it. + celery-worker: + condition: service_started + healthcheck: + test: + - CMD-SHELL + - >- + /opt/conda/envs/myenv/bin/python -c "import urllib.request as u; + r=u.urlopen('http://127.0.0.1:8008/api/health', timeout=5); + exit(0 if r.status == 200 else 1)" + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + deploy: + resources: + limits: + memory: 2g + + # Static website + edge proxy (nginx). TLS terminates in front of it, or + # replace nginx.prod.conf with a TLS variant (nginx.prod.tls.conf.example). + website: + build: + context: . + dockerfile: src/website/Dockerfile + args: + REACT_APP_BACKEND_HOST: ${PUBLIC_ORIGIN:?PUBLIC_ORIGIN is required} + REACT_APP_BACKEND_PORT: "" + restart: unless-stopped + logging: *default-logging + ports: + - "${HTTP_PORT:-80}:80" + depends_on: + backend: + condition: service_started + + # Nightly pg_dump into ${BACKUP_DIR} (default ./backups), pruned after + # BACKUP_RETENTION_DAYS, optionally copied off-host by BACKUP_OFFHOST_CMD. + backup: + image: pgvector/pgvector:pg16 + restart: unless-stopped + logging: *default-logging + entrypoint: ["/bin/bash", "/ops/backup-postgres.sh", "--loop"] + environment: + PGHOST: db + PGPORT: "5432" + PGUSER: ${DB_USER:?DB_USER is required} + PGPASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required} + PGDATABASE: ${DB_NAME:?DB_NAME is required} + BACKUP_DIR: /backups + BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-14} + BACKUP_INTERVAL_SECONDS: ${BACKUP_INTERVAL_SECONDS:-86400} + BACKUP_OFFHOST_CMD: ${BACKUP_OFFHOST_CMD:-} + volumes: + - ./scripts/ops:/ops:ro + - ${BACKUP_DIR:-./backups}:/backups + depends_on: + db: + condition: service_healthy + +volumes: + postgres_data: + redis_data: + redis_celery_data: + hf_cache: diff --git a/docker-compose.yml b/docker-compose.yml index 9793f34..e137585 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: volumes: - .:/app - ./src/database/init.sql:/docker-entrypoint-initdb.d/init.sql + - postgres_data:/var/lib/postgresql/data environment: - POSTGRES_PASSWORD=${DB_PASSWORD} - POSTGRES_USER=${DB_USER} @@ -115,6 +116,10 @@ services: - CELERY_BROKER_PORT=${CELERY_BROKER_PORT} - DB_HOST=db - DB_PORT=${DB_PORT} + - DB_NAME=${DB_NAME} + - DB_USER=${DB_USER} + - DB_PASSWORD=${DB_PASSWORD} + - TEST_DATABASE_URL=postgresql://postgres:postgres@test_db:5432/test_db - PYTHONPATH=/app/src - PYTHONMULTIPROCESSING_START_METHOD=spawn - TOKENIZERS_PARALLELISM=false @@ -179,12 +184,16 @@ services: - NVIDIA_DRIVER_CAPABILITIES=compute,utility - NVIDIA_REQUIRE_CUDA=cuda>=12.0 - SERVER_PORT=${SERVER_PORT} + - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS} - REDIS_HOST=redis - REDIS_PORT=${REDIS_PORT} - CELERY_BROKER_HOST=redis-celery - CELERY_BROKER_PORT=${CELERY_BROKER_PORT} - DB_HOST=db - DB_PORT=${DB_PORT} + - DB_NAME=${DB_NAME} + - DB_USER=${DB_USER} + - DB_PASSWORD=${DB_PASSWORD} - WEBSITE_HOST=website - WEBSITE_PORT=${WEBSITE_PORT} - PYTHONPATH=/app/src @@ -197,6 +206,20 @@ services: - CUDA_VISIBLE_DEVICES=0 - TORCH_CUDA_ARCH_LIST="8.6" - PRELOAD_MODELS=true + # Agent subsystem (ACP) upstream provider config, see .env.example + - AGENT_UPSTREAM_BASE_URL=${AGENT_UPSTREAM_BASE_URL} + - AGENT_UPSTREAM_API_KEY=${AGENT_UPSTREAM_API_KEY} + - OLLAMA_API_KEY=${OLLAMA_API_KEY} + - OPENAI_API_KEY=${OPENAI_API_KEY} + - GROQ_API_KEY=${GROQ_API_KEY} + - OPENROUTER_API_KEY=${OPENROUTER_API_KEY} + - AGENT_REASONING_EFFORT=${AGENT_REASONING_EFFORT} + - AGENT_RESPONSES_MAX_OUTPUT_TOKENS=${AGENT_RESPONSES_MAX_OUTPUT_TOKENS:-} + # Lets AGENT_UPSTREAM_BASE_URL=http://host.docker.internal:11434/v1 reach an + # Ollama instance running on the host from inside this container (Docker + # Desktop maps this automatically; Linux needs this explicit mapping). + extra_hosts: + - "host.docker.internal:host-gateway" depends_on: db: condition: service_healthy @@ -250,29 +273,27 @@ services: website: condition: service_healthy healthcheck: - test: ["CMD", "curl", "-f", "http://localhost"] + test: ["CMD", "curl", "-f", "http://localhost:8000"] start_period: 0s interval: 60s timeout: 5s retries: 3 -# test_db: -# image: pgvector/pgvector:pg16 -# container_name: postgres_test -# ports: -# - "5433:5432" -# environment: -# - POSTGRES_PASSWORD=postgres -# - POSTGRES_USER=postgres -# - POSTGRES_DB=test_db -# volumes: -# - .:/app -# - ./src/database/init_test.sql:/docker-entrypoint-initdb.d/init_test.sql -# healthcheck: -# test: ["CMD-SHELL", "pg_isready -U postgres"] -# interval: 5s -# timeout: 5s -# retries: 5 + test_db: + profiles: ["test"] + image: pgvector/pgvector:pg16 + container_name: postgres_test + ports: + - "5433:5432" + environment: + - POSTGRES_PASSWORD=postgres + - POSTGRES_USER=postgres + - POSTGRES_DB=test_db + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d test_db"] + interval: 2s + timeout: 5s + retries: 10 volumes: postgres_data: diff --git a/docs/AGENT_CONFIGURATION_CONTRACT.md b/docs/AGENT_CONFIGURATION_CONTRACT.md new file mode 100644 index 0000000..bb6c841 --- /dev/null +++ b/docs/AGENT_CONFIGURATION_CONTRACT.md @@ -0,0 +1,110 @@ +# Agent configuration contract (ISSUE-03) + +Recorded: 2026-09-19. This note records what the current release enforces and +what remains open, so a study arm's label cannot silently disagree with what runs. + +## Enforced today + +- **Framework ↔ release distribution mode.** A profile's `framework_version` and + its pinned release are validated together at profile create/update and again + when a study freezes its profile selection + (`research/study/agents/distributions.py::validate_profile_configuration`): + `code4me2-agent` requires a `PACKAGED` release; `goose`/`codex` require a + `BYOA_EXTERNAL` release with a declared command/package identity. A mismatch is + a typed 4xx (`FRAMEWORK_DISTRIBUTION_MISMATCH`) before enrollment. +- **Qualification.** The release must be `QUALIFIED` (derived from conformance + evidence bound to the exact artifact/platform — ISSUE-10); withdrawn or + unqualified releases are refused (`RELEASE_NOT_QUALIFIED`/`RELEASE_WITHDRAWN`). +- **Approval-option evidence.** The profile's approval option must be covered by + the release's passing conformance cases (`APPROVAL_OPTION_UNVERIFIED`). +- **Tools.** The selected tools must belong to the framework's catalogue + (`TOOLS_NOT_SUPPORTED`). +- **Packaged binaries resolve per release + platform.** The bootstrap pins the + assigned release's archive for the participant's platform; the plugin installs + exactly that archive (verified cache or its exact GitHub Release), and the + digest pin check remains fail-closed (no PATH fallback). +- **BYOA executable configuration (ISSUE-03 Path A).** A qualified BYOA release + declares `byoa_config`: one binding per governed profile field + (`model`, `temperature`, `max_steps`, `tools`, `approval_policy`), each with a + transport (`env` → environment variable, `arg` → argv pair), an optional + `value_map` (server vocabulary → agent vocabulary) and a list `format` + (`csv`/`json`). Profile/study creation rejects any profile field the release + does not translate (`BYOA_CONFIG_UNMAPPED`) and any list binding without a + list format (`BYOA_CONFIG_FORMAT_INVALID`). The bootstrap manifest projects + the bindings plus the frozen profile fields; the plugin applies them to the + BYOA launch (`ByoaConfiguration.kt`: mapped argv + `--agent-env KEY=VALUE` + overrides) and refuses to launch if a set field has no binding. The proxy + passes the overrides only to the agent child environment. +- **Provider connection and the research inference gateway.** The provider + connection itself is never translated: the server-held key stays on the + server. A Goose release (a *gateway-bound* framework, see + `INFERENCE_GATEWAY_FRAMEWORKS`) must additionally declare five *runtime* + bindings the plugin fills at launch — `inference_gateway_host` + (the research server origin), `inference_gateway_base_path` + (`api/research/inference/v1/chat/completions`), `inference_gateway_credential` + (the participant's inference capability; `env` transport only, never argv), + `provider_kind` (`openai_compatible`, translated through `value_map`, e.g. to + Goose's `openai`) and `state_dir` (a plugin-owned isolated agent state + directory). They are never profile fields. A Goose profile pinned to a release + without them is refused (`INFERENCE_GATEWAY_UNBOUND`) at profile/study + creation, at recipe import and at bootstrap. The bootstrap manifest then carries + an `inference_gateway` block with a signed, scoped inference capability + (audience `inference`, scope `inference:relay`), the agent's model calls go + through `POST /api/research/inference/v1/chat/completions`, and every call is + metered against the participant's budget. Codex releases are not gateway-bound: + they sign in with ChatGPT and are neither relayed nor metered. +- **Tools enforced at the gateway (`gateway` transport).** Goose reads no tool + selection from its environment: it offers the model every tool of its enabled + extensions on every call (Goose 1.51: `shell`, `edit`, `write`, `tree`, + `analyze`, `read_image`, `load`, `load_skill`, `delegate`, `todo__todo_write`, + `apps__*`, `extensionmanager__*`). A Goose release may therefore bind `tools` + as `{"field": "tools", "transport": "gateway", "key": "tool_allowlist", + "format": "json"}`. The plugin sets nothing for it, and the research inference + gateway enforces the frozen profile's selection on both sides of the call: + - **request:** the model is offered only the selected tools. An empty + selection means no tools; a `tool_choice` naming a withheld tool is dropped + or narrowed, and `parallel_tool_calls` is dropped with the last tool. + - **response:** a call the model still makes to a withheld tool is removed + from the body or stream before Goose sees it. The stream is read line by + line as Goose reads it and filtered by Goose's own assembly rules + (`agents/tool_call_filter.py`), so a withheld call cannot be rebuilt from + later pieces; a line the filter cannot judge (unreadable, or shaped so that + only Goose could read it) is dropped. When a turn's calls are all withheld, + the participant sees a short notice. Removals and dropped lines are logged. + + Tool names match exactly. A selection that matches none of the offered tools + (an agent build that renamed them) is logged as a warning. A release that + does not declare the binding keeps Goose's own tool definitions untouched. + + Only a release that provably launches a gateway-bound agent can declare it: + its identity must name only Goose and its command (else package) must be + Goose's own executable. Anything else, a Codex release or a mis-declared one, + is refused (`BYOA_CONFIG_UNENFORCEABLE`) and never offers `tools` as + configurable. The plugin counts the binding only when the manifest carries an + inference gateway. + + The binding is opt-in per release, and it is not in the canonical example: + profiles written before it default to `tools_json = "[]"`, which would become + no-tools arms. Plugins older than this contract refuse such arms, failing + closed, so ship the plugin first. + +## Explicit seams + +- The participant plugin bundles no agent: `PackagedAgentInstaller` installs + the assigned release's archive for the platform from the verified cache or its + exact GitHub Release (SHA-256 pinned by the bootstrap), or the activation + blocks (`RUNTIME_UNAVAILABLE` when retryable, otherwise `PREPARATION_FAILED`). + PATH is never consulted. +- Real Goose/Codex key names are release-owned data (the `byoa_config` + bindings), not plugin constants. Conformance evidence for a release that + declares a mapping remains part of release qualification (ISSUE-10/04); the + plugin-side tests use a recording double to assert the declared mapping is + applied to argv/env exactly. + +## Deliberate decision + +The reverted blanket study-creation guard (`feffcd4`/`157337d`) is not +reintroduced. Rejections are per-combination (framework × distribution mode × +qualification × approval evidence × field coverage), so a legitimately qualified +BYOA study stays creatable while an unexecutable combination fails early with a +typed error. diff --git a/docs/CONTENT_POLICY_DECISION.md b/docs/CONTENT_POLICY_DECISION.md new file mode 100644 index 0000000..c37978f --- /dev/null +++ b/docs/CONTENT_POLICY_DECISION.md @@ -0,0 +1,38 @@ +# Content-storage policy decision (ISSUE-01) + +Recorded: 2026-09-19. + +## Authority + +Content storage for a research-bound context is decided in exactly one place, +`research/telemetry/content_policy.py`, from: + +- the account's **ACTIVE enrollment** for the task's study, with accepted + consent, and +- the study's **frozen telemetry policy** (`research_config_json.telemetry_policy`) + resolved through the shared `PrivacyPolicy` engine. + +Missing or malformed policy denies. The legacy account preference +(`store_agent_content`) is *not* study consent and is only consulted for +genuinely non-research contexts (no study binding), where there is no study +policy to enforce. Every persistence boundary — managed-agent ingest, the +plugin/proxy task paths, the legacy relay adapter, and the ACP policy endpoints +— resolves through this authority. + +## `code_metadata_mode` + +The study policy controls how code metadata is persisted: + +- **`hash` (default)** — code metadata is stored as a SHA-256 token. This is the + mode when the policy does not explicitly allow `CODE_METADATA` (including when + no policy is declared at all for a non-research context). +- **`allow`** — the raw value is stored, only when the frozen policy's + `allowed_field_classes` explicitly contains `CODE_METADATA`. + +The server never silently rewrites already-stored research data. The ingestion +writer validates the canonical event against the policy and **rejects** a +non-compliant event (`PRIVACY_BLOCKED`) instead of editing its bytes, because a +rewrite would invalidate the producer's digest and turn a retry into an +integrity conflict. Producers are expected to filter before sending; the +canonical adapter builds content-bearing fields only when the resolved policy +allows content. diff --git a/docs/MANAGED_AGENT_DEPLOYMENT.md b/docs/MANAGED_AGENT_DEPLOYMENT.md new file mode 100644 index 0000000..b630ff8 --- /dev/null +++ b/docs/MANAGED_AGENT_DEPLOYMENT.md @@ -0,0 +1,27 @@ +# Managed agent deployment + +The participant plugin assumes a globally reachable HTTPS backend and managed ACP protocol v1. Provider credentials belong only in the backend environment; participant machines must not receive them. A Goose arm's machine receives only a scoped, revocable *inference capability* (a bearer for the research inference gateway that is re-checked on every call against the enrollment, the study window, the kill switch and the participant's budget), never the provider key. + +## Deployment order + +1. Back up the study database. +2. Deploy the candidate backend with traffic disabled or readiness failing. +3. Run `python src/database/migration/migration_manager.py migrate` as a one-shot deployment job. In the Compose image, use `/opt/conda/envs/myenv/bin/python src/database/migration/migration_manager.py migrate`; the command now loads the same non-overriding `.env` fallback as the application and exits unsuccessfully unless the database reaches the code's exact Alembic head. +4. Confirm the database is at the expected Alembic head and the managed ACP capabilities endpoint reports ready. +5. Run the managed agent smoke test through the public HTTPS origin. +6. Enable traffic, then distribute the plugin ZIP built for this protocol and backend version. + +Do not reset a deployed study database during an upgrade. A failed migration must stop deployment and leave the existing application version serving until the database is restored or the migration is corrected. + +## Release checks + +- Upgrade a copy of the previous deployed schema and initialize an empty database. +- Confirm the active study contains only runtime profiles certified for the participant release. +- Confirm the backend holds every provider key named by an active profile. +- Confirm every metered model (Goose and built-in arms) has a price on its provider connection and every active study has a non-zero default participant budget; without them the arms refuse every call (`503 price_missing` / `402 quota_exhausted`). Apply the SQL in `docs/research-platform/RELEASES.md` to an existing database first. +- Confirm the Goose release in the catalogue binds the inference gateway (the release catalogue shows `inference_gateway: true`); an unbound release blocks bootstrap with `INFERENCE_GATEWAY_UNBOUND`. +- Verify grant exchange, run creation, inference, telemetry, logout revocation, and expiry recovery. +- Block direct provider access on the participant test machine and confirm inference still succeeds — for a Goose arm through the research inference gateway, and confirm the study's budget ledger shows the settled call. +- Scan the plugin and runtime artifacts for `.env` files, provider credentials, source-checkout paths, and `local-dev` references. + +Record the server commit, Alembic head, plugin commit, runtime version, runtime checksums, IDE version, and tested operating systems with each release. diff --git a/docs/PROTOCOL_VOCABULARY_DECISION.md b/docs/PROTOCOL_VOCABULARY_DECISION.md new file mode 100644 index 0000000..8c082b1 --- /dev/null +++ b/docs/PROTOCOL_VOCABULARY_DECISION.md @@ -0,0 +1,100 @@ +# Protocol vocabulary decision: `condition_id` and the revision-era validators + +- Status: recorded 2026-09-19 (ISSUE-19 item 1; review §12.3(6)/§14.3). +- Owner: `code4me2-server` server/protocol maintainers. +- Review trigger: the next protocol cleanup after ISSUE-19. Every allowlisted + symbol must then be deleted, or re-justified with a fresh owner and trigger. + +## Decision + +The remaining `condition_id` protocol-authoring vocabulary +(`research.study.protocol.models.StudyCondition`, +`research.study.protocol.validation` condition uniqueness/weights, and +`research.study.protocol.canonical` condition ordering) has **no execution +consumer** in the revision-free study lifecycle. + +It is kept (not deleted) and stays on the forbidden-symbol allowlist as a +temporary retirement record, with an owner and a planned-removal review trigger +per entry (`scripts/dev/forbidden_symbols.txt`). The validation module is not +deleted: it still backs the authoring-only protocol contract, and removing it +before the current profile/study invariants were demonstrated independently +would have been unjustified. The scan keeps the vocabulary from spreading into +new code: any occurrence outside the three allowlisted files fails closed. + +## Current invariants are enforced independently + +The current `create_study()` / clone / profile invariants do not pass through the +revision-era validators: + +- profile owner, active state, release presence, released status + (`RETIRED`/`BLOCKED` withdrawn, `QUALIFIED` required) and executable + framework/mode/tools/approval-evidence binding: + `research.study.protocol.store.build_profile_selections` (shared by study + create and stopped-study clone) and + `research.study.agents.distributions.validate_profile_configuration`. +- qualification is derived from verified conformance evidence bound to the exact + artifact/platform: `research.study.agents.registry.derive_qualification_status` + and `qualified_artifact_keys`, plus the bootstrap artifact-selection gate. + +## Test coverage (ISSUE-03 / ISSUE-10 / ISSUE-17) + +- ISSUE-03: + `tests/backend_tests/research/test_distributions.py::test_profile_configuration_matrix`, + `::test_profile_configuration_enforces_approval_option_evidence`, + `tests/backend_tests/research/test_profile_release_contract.py::test_create_profile_rejects_a_framework_release_mode_mismatch`, + `::test_create_profile_accepts_a_qualified_packaged_release`, + `::test_create_profile_accepts_a_qualified_byoa_release`. +- ISSUE-10: + `tests/backend_tests/research/test_runtime.py::test_qualification_is_derived_from_passing_conformance_evidence`, + `::test_cross_artifact_receipt_does_not_qualify_either_artifact`, + `::test_qualified_artifact_keys_bind_only_the_receipts_own_platform`, + `::test_bootstrap_selects_only_the_artifact_the_evidence_qualifies`, + `::test_bootstrap_qualifies_a_byoa_release_bound_to_its_manifest_digest`. +- ISSUE-17: + `tests/backend_tests/research/test_distributions.py::test_release_only_byoa_distribution_derives_mode_and_identity_from_release`, + `::test_release_only_packaged_distribution_is_not_reported_as_byoa`. +- Study-creation freeze: + `tests/database_tests/test_research_schema_lifecycle.py::test_study_creation_freezes_owned_profiles_and_rejects_foreign_profiles`. + +## Tested boundary: qualification vs verification vs readiness (ISSUE-004) + +These three concepts stay distinct in code and docs; no blanket restriction +was added: + +- **Qualification** (release-level, receipt-derived): a BYOA release is + `QUALIFIED` exactly when a passing conformance receipt binds its own + `source_manifest_digest` plus the adapter digest + (`research.study.agents.registry.qualified_artifact_keys` / + `byoa_identity_qualified`; ISSUE-003 evidence, read-only here). +- **Verification** (distribution-level, publication-time): a `BYOA_EXTERNAL` + distribution is always unverified — `resolve_distribution_view` reports + `verified=False` — so publication emits `DISTRIBUTION_UNVERIFIED` as an + admin `WARNING` (publishable) and a non-admin `ERROR` (blocked). The + admin/non-admin paths are preserved, not weakened. +- **Readiness** (participant-host, resolution-time): a host without the + installed agent blocks with `AGENT_NOT_FOUND`; the plugin resolves a + present agent through the documented discovery order (configured command, + release command, `PATH`, known locations). + +Role-severity proof (ISSUE-004, resolver-level, no database): +`tests/backend_tests/research/test_profile_release_contract.py::test_byoa_resolver_resolves_receipt_qualified_identity_but_not_unqualified`, +`::test_byoa_publication_severity_is_role_specific`. +Participant-flow proof (DB-backed HTTP): +`tests/backend_tests/research/test_study_lifecycle_bootstrap_session_telemetry_http.py::test_http_codex_byoa_adapter_normalization_and_terminal_closure` +(qualified BYOA release → study create → bootstrap manifest with the BYOA +identity). +Plugin discovery proof: +`code4me2/src/test/kotlin/me/code4me/research/proxy/ProxyTest.kt::ByoaAgentResolverTest`. +The `MANAGED_RUNTIME_FRAMEWORK` note in +`research.study.agents.enums` records the same boundary as a comment only; +the vocabulary itself is unchanged (additive-only). + +## Allowlist rules + +- Each entry must name an owner and a planned-removal review trigger. An entry + without both is a review failure. +- The scan must report `0 forbidden`; a new hit outside the allowlist fails the + build. Only a per-symbol entry is acceptable for a multi-symbol file. +- Deleting the symbol is preferred over extending the allowlist. When the + authoring module's last consumer disappears, the entry and the code go + together. diff --git a/docs/RESEARCH_ANALYTICS_SCOPE.md b/docs/RESEARCH_ANALYTICS_SCOPE.md new file mode 100644 index 0000000..89a9207 --- /dev/null +++ b/docs/RESEARCH_ANALYTICS_SCOPE.md @@ -0,0 +1,105 @@ +# Research analytics scope + +This note records the scope boundaries of the research analytics read paths and +the assignment-model protocol decisions (ISSUE-13, then the 2026-10 assignment +policy). + +## Assignment model (protocol decision) + +Assignment is **enrollment-owned, equal-probability, sticky random**, and every +study freezes how it draws in its configuration +(`research_config_json["assignment"]`, part of the configuration digest): + +- The unit of randomization is the `enrollment_id` (never account, device, task + or process); it is a server-generated UUID created when the participant + consents, so nobody can choose or re-roll it. +- **Studies created since the 2026-10 decision** draw with + `DETERMINISTIC_HASH` (`research.runtime.assignment.hashing`). With `s` the + study id, `e` the randomization epoch (0), `u` the enrollment id and the + study's K arms ordered by `selection_order`: + + h(u) = first 64 bits (big-endian) of SHA-256("s:e:u") + arm(u) = arm k with k = floor(K * h(u) / 2^64) + + UUIDs are lowercase and hyphenated. The draw is reproducible from exported + data, P(arm k) = 1/K up to an error below 2^-64, and the study id salts it, so + studies are independent. Golden vector: + `00000000-0000-0000-0000-000000000001:0:00000000-0000-0000-0000-000000000002` + gives h = 0x39673deb8a861ef7, so k = 0 (K = 2 or 3) and k = 1 (K = 5). +- **Studies created earlier** carry no `assignment` block and keep + `RANDOM_EQUAL`: an equal-probability draw from the system CSPRNG + (`secrets.choice` at join; `SystemRandom` in the bootstrap fallback). +- Both are **simple randomization**: arm sizes are Multinomial(N, 1/K), so small + studies are often unbalanced (two arms: P(larger arm >= 60%) is .50 at N = 20, + .31 at N = 24, .27 at N = 40 and .06 at N = 100). There is no blocking, + stratification or minimization. +- **Manual override (opt-in per study).** A study created with + `allow_manual_assignment` lets its owner set a participant's arm by hand + (`PUT /api/research/studies/{id}/enrollments/{enrollment_id}/assignment`), only + while the enrollment is active and before any research session, event, + metered inference reservation or agent task exists for it. The consent notice + of such a study says participants are assigned "at random or by the research + team". The row keeps its `randomization_epoch` and becomes `strategy = MANUAL`, + so the randomized arm stays recomputable (as-randomized vs. as-assigned + analyses); a `STUDY_LIFECYCLE` record (`ASSIGNMENT_OVERRIDDEN`) names the old + and new arm, and the session bootstrap reads the assignment under a share lock + so an override and a first session serialize. +- Otherwise the assignment is immutable and sticky: a repeated bootstrap/join + returns the existing assignment and never re-randomizes it + (`StudyAssignment`, unique `enrollment_id`). +- `AssignmentStrategy.STRATIFIED` / `WEIGHTED_RANDOM` remain additive protocol + vocabulary; `allocate` refuses any strategy other than `RANDOM_EQUAL` and + `DETERMINISTIC_HASH`. + +Changing how a study draws is a protocol decision recorded here, never a silent +change to `allocate` or the join path. + +## Personal agent analytics (unchanged) + +`research.analysis.read_models.dashboard` (the Dashboard → Agents tab) is scoped +to the caller's **own** agent tasks: a non-administrator only ever aggregates +rows where `agent_task.owner_user_id = current_user.user_id`; an administrator +may optionally scope to one user. It is personal run analytics, not participant +results, and this slice does not change it. + +## Study-owner participant analytics + +Study-owner-scoped participant coverage lives in: + +- Endpoint: `GET /api/research/operations/participants/coverage?study_id=` + (`src/backend/routers/research/read_models.py`). +- Read model: `StudyParticipantCoverageV1` / `ParticipantCoverageRowV1` + (`src/research/analysis/read_models/models.py`), built by + `build_participant_coverage` (`.../service.py`) from study-scoped domain + records loaded by `list_coverage_inputs` (`.../store.py`). + +Each row is keyed by the study-local `enrollment_id` / `participant_code` and +contains: + +- enrollment status and timestamps; +- frozen assignment facts (profile id, strategy, randomization epoch, profile + digest, status) — never the profile snapshot or any secret; +- session counts (total/active/terminal) plus `last_activity_at` / + `last_heartbeat_at`; +- canonical event counts total and grouped by event type and source, plus the + last event timestamp. + +Access control: only the study owner (`study.created_by`) or an administrator +may read it (`require_study_owner`), matching the existing enrollment/telemetry +coverage endpoints. Rows are always study-scoped, so no cross-study personal +timeline is ever exposed; login/account/participant identity is never joined +into the payload, and retention tombstones (`retention_state = "DELETED"`) are +excluded. + +## Participant budgets and metered spend + +Goose and built-in arms spend from the study's provider connection (the shared, +server-held key). Every enrollment owns one balance row (`limit`, `settled`, +`reserved`, micro-USD) and every metered call one reservation-ledger row. The +participants table carries each row's `budget` (limit, consumed, reserved, +remaining, exhausted) and the study summary's `totals` carry +`metered_spend_micro_usd` / `metered_calls` computed from the ledger (exact, +all-time, not windowed) — these are separate from the telemetry-derived +`usage_tokens`, which stay agent-reported. Participants see only their own +arm-blind numbers (`budget` on *My studies*): never a model, price or profile. +Codex arms are unmetered (`budget: null`). diff --git a/docs/research-platform/FROM_SCRATCH_SETUP.md b/docs/research-platform/FROM_SCRATCH_SETUP.md new file mode 100644 index 0000000..4276bd9 --- /dev/null +++ b/docs/research-platform/FROM_SCRATCH_SETUP.md @@ -0,0 +1,394 @@ +# From-scratch setup (dev) — end to end + +This walks through a working Code4Me research installation on a clean machine: +infrastructure → migrations → admin → researcher → provider → agent → plugin → +study → participant. Each step says what to do and what to expect. + +**Repository layout (required):** both repositories live side by side under one +workspace root. + +``` +/ + code4me2/ # IntelliJ plugin (ships no agent; installs the study's pinned one) + code4me2-server/ # backend + website + producer + e2e harness +``` + +**Roles:** *operator/admin* = infrastructure + catalogue + provider; *researcher* += profiles + studies; *participant* = enters the join code, installs the plugin, +starts the session. + +--- + +## 0. Prerequisites + +| Requirement | Note | +|---|---| +| Docker + Docker Compose v2 | for db/redis/backend/website/nginx | +| JDK 21+ | plugin build; the Gradle toolchain downloads JDK 25 itself when needed (foojay) | +| IntelliJ IDEA | to install the plugin and start a session | +| Python 3.12 venv (`code4me2-server/.venv`) | agent builds + e2e harness | +| PyInstaller | `uv pip install -r packaging/requirements-build.lock` (documented build dependency) | + +> Use `code4me2-server/.venv`. A workspace-root `.venv` may have a broken pytest +> (`ImportError: Deque`) and a full agent build then fails at its test step. + +--- + +## 1. Environment file (`.env`) + +`code4me2-server/.env` (gitignored) holds the required values. The critical ones: + +```dotenv +DB_PORT=5433 # host port (same port inside compose) +DB_NAME=code4meV2 +DB_USER=postgres +DB_PASSWORD=postgres +SERVER_PORT=8008 +OPENROUTER_API_KEY=... # your provider's key (never written to the database) +``` + +- Provider keys live **only** here; the database stores the *name* (`secret_ref`) + and the backend resolves the value from its own environment. Goose and + built-in study arms spend from this one key through a metered relay; the + optional `INFERENCE_*` knobs in `.env.example` tune the hold estimate, output + cap, reservation deadline and the inference capability lifetime. +- `CODE4ME_DEV_RELOAD=1` (dev compose default): the backend restarts on source + changes, so a long-running stack never serves stale in-memory code. + +--- + +## 2. Bring the stack up + +```bash +cd code4me2-server +docker compose -f docker-compose.dev-arm.yaml up -d --build +docker compose -f docker-compose.dev-arm.yaml ps +``` + +Expected: `db`, `redis`, `redis-celery`, `backend`, `celery-worker`, `website`, +`nginx` → **healthy**. + +| Address | What | +|---|---| +| http://localhost:8008/docs | Backend API (OpenAPI) | +| http://localhost:3000 | Website (login/signup/dashboard/join) | +| localhost:5433 | Postgres (db: `code4meV2`) | + +> **Data persistence:** Postgres data lives in the `pgdata` named volume. +> `docker compose -f docker-compose.dev-arm.yaml down` removes the containers but +> **keeps** the data. Use `down -v` deliberately when you want a full reset (it +> also drops the `website_node_modules` volume), then rerun the migrations in +> step 3. +> +> Without the volume, `down` would destroy the database: `init.sql` only creates +> the 22 base tables and the research schema arrives via migrations. With no +> schema, login answers `500 Server failed to authenticate the user!` and the +> `user` table is empty. + +--- + +## 3. Migrations + +`init.sql` runs automatically only on an **empty volume** (base tables). The +research schema comes from migrations: + +```bash +docker exec backend bash -lc "source activate myenv && cd /app && \ + python src/database/migration/migration_manager.py status" +docker exec backend bash -lc "source activate myenv && cd /app && \ + python src/database/migration/migration_manager.py migrate" +``` + +Expected `status`: `Database: Connected` · `Tables: 45` · `At expected head`. + +There is a single revision, `8a0084080b46_consolidated_schema.py`. It **refuses +to run** against a database that still holds the previous research tables — use a +new database/volume (never reset a production database in place). + +--- + +## 4. First admin (one-time) + +```bash +curl -X POST http://localhost:8008/api/user/create \ + -H 'Content-Type: application/json' \ + -d '{"config_id":1,"email":"admin@example.com","name":"Admin","password":""}' +``` + +```sql +-- one-time, operator-run; there is deliberately no admin-granting endpoint +UPDATE "user" SET is_admin = TRUE WHERE email = 'admin@example.com'; +``` + +Verify: http://localhost:3000/login → sign in with that account → the dashboard +shows the **ADMIN VIEW** badge and the `Accounts / Provider Connections / Agent +Catalogue` tiles. + +--- + +## 5. Researcher account + +1. Create an account at `/signup` (participants and researchers use the same path). +2. Grant researcher access as the admin: + - **UI:** Dashboard → **Accounts** → tick **Researcher** on that row. + - **API:** `PUT /api/research/researchers/` → `{"can_research": true}` + +`can_research` is the single researcher authority (there is no per-study role +table). Administrators bypass ownership checks. + +--- + +## 6. Provider connection (+ secret) + +**UI:** Dashboard → **Provider Connections** + +| Field | Example | +|---|---| +| Label | `prod-openrouter` | +| Base URL | `https://openrouter.ai/api/v1` | +| Secret name (env var) | `OPENROUTER_API_KEY` ← the **name**, not the value | +| Models (one per line) | `cohere/north-mini-code:free` | + +After saving, the row must show **"Ready — secret present in deployment"** (the +named env var exists and the backend can read it). + +**Model prices (required for Goose and built-in arms):** in the same editor, +enter each model's price in **USD per million tokens** (input, output, and +optionally cached input). A metered arm whose model has no price refuses every +call with `503 price_missing`, and the study form warns before creation. The +connection card shows **"Prices: N of M models"** and a *price missing* chip per +unpriced model. + +**API:** `POST /api/research/provider-connections` → `{label, base_url, secret_ref, models, is_active, model_prices?}` +(`model_prices` = `{model: {input_usd_per_million, output_usd_per_million, cached_input_usd_per_million?}}`; omitted = unchanged) + +Selection rule: every **active** connection can be selected in any signed-in +user's profile (there is no per-user grant flow). The secret value never enters +the database. + +--- + +## 7. Produce the agent (release) + +```bash +cd code4me2-server + +# A) Full build: stamp the version → runtime tests → PyInstaller → archive → +# --version / --self-check / ACP initialize against the packaged executable +PYTHONPATH=src .venv/bin/python -m research.study.agents.participant_release native \ + --version 1.2.3 --platform macos-arm64 \ + --server-commit "$(git rev-parse HEAD)" \ + --output dist/release-1.2.3 + +# B) Re-test an already built bundle (the version must equal the bundle's) +PYTHONPATH=src .venv/bin/python -m research.study.agents.participant_release native \ + --skip-build --version 1.2.3 --platform macos-arm64 \ + --server-commit "$(git rev-parse HEAD)" --output dist/re-test-1.2.3 +``` + +Rules: +- Run the command **from `code4me2-server`** (that is where `src/research` lives). +- The `--output` directory must **not** exist yet (immutable release rule). +- With `--skip-build`, `--version` must equal the version embedded in the bundle. +- Platforms: `macos-arm64`, `macos-x64`, `linux-x64`, `windows-x64` (`aarch64` is normalised to `arm64`). +- **Failed tests produce no output directory.** + +Output: +``` +dist/release-1.2.3/code4me-agent-macos-arm64.zip +dist/release-1.2.3/native-macos-arm64.json ← tests: self_check/acp_initialize PASS +``` + +> **Keep this ZIP.** Those bytes are what a study pins and what the IDE's agent +> cache must hold (step 9). Rebuilding the same version can produce a different +> sha → the pin no longer matches. + +--- + +## 8. Import the agent into the catalogue + +**UI:** Dashboard → **Agent Catalogue** + +- From files: paste `native-macos-arm64.json` into `Runtime manifest JSON` → + upload the **complete set** of ZIPs the manifest declares via `Agent archive + files` → **Verify and import**. +- From URLs: **Import from release URLs** → `Manifest URL` + `Archive URLs` (one + ZIP per line). + +**API:** +```http +POST /api/research/agents/releases/import # multipart: manifest (text) + archives (files) +POST /api/research/agents/releases/import-url # {manifest_url, archive_urls[]} +``` + +What the server verifies: the exact declared set (missing/extra/duplicate), +sha256 + size (computed server-side) and the test results. Any failure rejects +the **whole** import. + +Result: a `release_id` (e.g. `code4me-agent-1.2.3-9f5f4e5d1c53`) and **QUALIFIED** +status — derived from the platform test results; there is no separate +approval/receipt step. + +**Emergency stop:** **Disable release** in the release detail is one-way; +re-importing the same manifest does not re-enable it. + +--- + +## 9. Build and install the plugin, then give the IDE the agent + +The plugin bundles **no agent**. A study pins its agent archive by SHA-256; the +plugin uses a verified entry of its agent cache or, on a cache miss, downloads the +archive from the exact GitHub Release URL the imported manifest carries. A locally +produced release has no public URL, so place the imported ZIP in the IDE's agent +cache yourself. + +**Critical rule:** the cached ZIP must be **the ZIP that was imported into the +catalogue** — not a fresh build. Its SHA-256 names the cache file and is verified +again before use; other bytes block the study. + +```bash +cd code4me2 +./gradlew buildPlugin # output: build/distributions/client-.zip + +# The IDE's system directory; on macOS usually: +SYSTEM="$HOME/Library/Caches/JetBrains/IntelliJIdea2026.2" +ZIP=../code4me2-server/dist/release-1.2.3/code4me-agent-macos-arm64.zip +SHA=$(shasum -a 256 "$ZIP" | cut -d' ' -f1) +mkdir -p "$SYSTEM/code4me/runtimes/code4me-agent/archives" +cp "$ZIP" "$SYSTEM/code4me/runtimes/code4me-agent/archives/$SHA.zip" +``` + +**Install:** IntelliJ → **Settings → Plugins → ⚙ → Install Plugin from Disk…** → +ZIP → **Restart IDE**. + +The dev backend URL is already `http://localhost:8008` in +`src/main/resources/plugin.conf`. + +`scripts/build-plugin-with-agent.py` still bundles a release into the plugin, but +only the ordinary (non-study) managed setup reads that bundle; studies never do. + +> The plugin version (`pluginVersion` in `gradle.properties`) and the agent +> version (`--version` at production) are independent axes. The runtime compares +> the agent's **sha256**, never its version string. + +--- + +## 10. Profile → study → participant + +**Profile (researcher):** Dashboard → **Agent Profiles** + +| Field | Value | +|---|---| +| Profile name | `default-code4me2-agent` | +| framework | `code4me2-agent (built-in)` | +| connection_id | the connection from step 6 | +| model | one of the connection's models | +| release_id | the release created in step 8 (must be QUALIFIED) | +| approval_policy | `Auto-approve` / `Ask per step` / `Suggestion only` | +| tools, max_steps, temperature | from the framework catalogue / limits | + +**Study (researcher):** Dashboard → **Research Control Plane** → **New study** → +name/dates → select the profile → **default budget per participant (USD)** → +telemetry policy + session policy → create → **Join code**. + +The default budget is required (> 0) whenever a selected profile runs Goose or +the built-in agent (Codex arms sign in with ChatGPT and are not metered). It is +editable at any time in the study's **Settings → Participant budgets** card, +which also applies a new default to the participants still on the old one; the +**Participants** tab shows spent/budget per participant and the drawer's +**Adjust budget** tops up or sets an individual limit (with a reason). +Participants see "Budget remaining" on *My studies*. Every enrollment is born +with a balance at the study default; a participant whose budget is used up gets +`402 quota_exhausted` from the agent until topped up — nothing else changes. + +**Participant:** +1. Account at `/signup` (or an existing account). +2. Web: **Join a research study** → code → study preview → **consent** + (or plugin: **Join a Study** → code). +3. Open the project in IntelliJ → start the session from the Code4Me research + settings → pick **Code4Me Research Proxy** in AI Chat. +4. The plugin installs the pinned agent from its cache (run with `--managed`) + and session telemetry flows. + +--- + +## 11. Verification and shortcuts + +```bash +# Whole backend flow (disposable stack; real import + session + telemetry + revoke) +cd code4me2-server && ./e2e/test --layer backend + +# Admin panel + study + join flow (real browser) +CODE4ME_FLOW_MANIFEST=$PWD/dist/release-1.2.3/native-macos-arm64.json \ + node e2e/browser/admin_flow.js + +# Backend + database tests (isolated DB; never touches the dev database) +docker exec postgres createdb -U postgres code4me_release_refactor_test 2>/dev/null || true +docker exec -e TEST_DATABASE_URL=postgresql://postgres:postgres@db:5433/code4me_release_refactor_test \ + backend sh -c 'cd /app && /opt/conda/envs/myenv/bin/python -m pytest tests/backend_tests/research tests/database_tests -q' + +# Website +cd src/website && CI=true npm test -- --watchAll=false --runInBand && CI=true npm run build + +# Plugin test suite +cd ../../../code4me2 && ./gradlew test +``` + +**Dev seed (shortcut):** catalogue + pin + study in one command from a produced manifest: +```bash +MANIFEST=dist/release-1.2.3/native-macos-arm64.json scripts/dev/seed_local_dev.sh +``` +(It rejects a manifest without test results and prints the producer command.) + +--- + +## 12. Troubleshooting + +| Symptom | Meaning | Fix | +|---|---|---| +| `No accounts found.` | The running backend may hold stale code | `docker restart backend` (dev reload is on; make sure `CODE4ME_DEV_RELOAD=1`) | +| `ModuleNotFoundError: No module named 'research'` | Command run from the wrong directory | `cd code4me2-server` (that is where `src/research` lives) | +| `packaged executable version does not match the release` | `--skip-build` with a different version | Match the bundle's version, or drop `--skip-build` | +| `TESTS_NOT_PASSED` | The manifest carries no passing platform test | Produce the manifest with the producer command | +| `DIGEST_MISMATCH` / `SIZE_MISMATCH` | The uploaded bytes disagree with the manifest | Upload the ZIP from the same build | +| `ARTIFACT_MISSING` / `UNEXPECTED_ARCHIVE` / `DUPLICATE_ARCHIVE` | Archive set is not exactly what the manifest declares | Upload the full set, one copy each | +| `RELEASE_NOT_QUALIFIED` (profile) | Release has no test results, or was disabled | Import a tested manifest; `DISABLED` cannot be re-enabled | +| `FRAMEWORK_DISTRIBUTION_MISMATCH` | Profile framework does not match the release mode | `code4me2-agent` ↔ PACKAGED; Goose/Codex ↔ BYOA | +| `The study's agent has no public GitHub Release URL…` | The release has no download URL (a local release) and its ZIP is not in the IDE's agent cache | Step 9: copy the imported ZIP into the cache under its SHA-256 | +| `This study has no agent for /` | The imported release does not declare that platform | Step 8: produce and import the release for the platform | +| `unrecognized arguments: --status-file / --agent-run-id / --telemetry-policy-digest` | The packaged proxy bundle is stale (no such CLI flags) | `./gradlew buildResearchProxyBundle`, then rebuild/reinstall the plugin | +| `ARTIFACT_UNAVAILABLE` (bootstrap) | The imported release has no archive for the participant's platform | Produce and import the release for that platform (steps 7–8) | +| `401` from the provider | `secret_ref` env var is empty/wrong | Put the key in `.env`, `docker restart backend` | +| `402 quota_exhausted` in the agent chat | The participant's budget is used up (or the study default is still 0) | Study → Settings → Participant budgets (set/apply the default) or the participant drawer → Adjust budget | +| `503 price_missing` in the agent chat | The arm's model has no price on its provider connection | Provider Connections → enter the model's prices | +| `INFERENCE_GATEWAY_UNBOUND` (bootstrap / profile) | The Goose release does not bind the research inference gateway | Re-import a recipe whose Goose agent declares the runtime bindings (see `docs/participant-release.example.json`) and re-pin the study | +| `BUDGET_REQUIRED` / `BUDGET_PRICE_MISSING` (create study) | A selected Goose/built-in profile needs a default budget / a priced model | Enter the default budget; price the model on its connection | +| Migration "refuses to run" | The database still holds old research tables | Use a new database/volume (no in-place production reset) | + +--- + +## 13. Do not + +- Write a provider key into the database, the plugin or a manifest (store the *name* only). +- Reset/drop tables in a production database. +- Distribute the developer `buildPlugin` ZIP to participants (the participant + release is a separate CI flow). +- Try to revive a `DISABLED` release by re-importing it. +- Delete a produced release ZIP: a study's pin is bound to those bytes and a + rebuild yields a different sha. + +--- + +## 14. Common paths (summary) + +``` +Producer: code4me2-server/dist/release-/{native-.json, code4me-agent-.zip} +Catalogue: Dashboard → Agent Catalogue → Verify and import +Provider: Dashboard → Provider Connections +Profile: Dashboard → Agent Profiles +Study: Dashboard → Research Control Plane +Join: /join (web) or the plugin's "Join a Study" +Plugin: code4me2/build/distributions/client-.zip +Dev seed: scripts/dev/seed_local_dev.sh (with MANIFEST=...) +E2E: ./e2e/test --layer backend · CODE4ME_FLOW_MANIFEST= node e2e/browser/admin_flow.js +``` diff --git a/docs/research-platform/PRODUCTION_DEPLOYMENT.md b/docs/research-platform/PRODUCTION_DEPLOYMENT.md new file mode 100644 index 0000000..0fd259f --- /dev/null +++ b/docs/research-platform/PRODUCTION_DEPLOYMENT.md @@ -0,0 +1,151 @@ +# Production deployment + +How to run the Code4Me research platform for a live study. Written for the +operator who deploys the server; the plugin side is in +`code4me2/docs/PARTICIPANT_SETUP.md`. The stack is `docker-compose.prod.yml`; +everything else in the repository root (`docker-compose.yml`, +`docker-compose.dev-arm.yaml`) is a GPU or development stack and must not serve +participants. + +## Topology + +``` +participants (IDE plugin, browser) + │ https:// + ▼ + TLS edge (cloud LB / Caddy / Traefik / host nginx) ← terminates TLS, forwards to :80 + │ + website (nginx) ─ static React build, proxies /api/ → backend:8008 + │ + backend (uvicorn, N workers) ── db (Postgres 16 + pgvector, 127.0.0.1:5432 on the host) + │ ── redis (AOF, password) ── redis-celery (AOF) + │ ── celery-worker (classic rows) + migrate (one-shot, runs before backend) backup (nightly pg_dump → ./backups) +``` + +Only the `website` container is published (`HTTP_PORT`, default 80). Postgres is +bound to `127.0.0.1` for `psql`/restores. Redis and the backend are reachable +only inside the compose network, which is why the backend trusts +`X-Forwarded-For` from any peer (`FORWARDED_ALLOW_IPS=*`): the only peer is +nginx, and nginx forwards exactly one hop (`$remote_addr`, never the +client-supplied chain), so a client cannot choose its own rate-limit key. +Behind an external TLS edge `$remote_addr` is the edge: uncomment the +`set_real_ip_from`/`real_ip_header` lines in `nginx.prod.conf` with the edge's +address so nginx recovers the real client first. + +### TLS + +Terminate TLS in front of the `website` container and forward plain HTTP to +`HTTP_PORT`, preserving `X-Forwarded-Proto: https` (Caddy, Traefik and cloud +load balancers do; for a host nginx add `proxy_set_header X-Forwarded-Proto +$scheme;`). Cookies are issued with `Secure` (`COOKIE_SECURE=true`) and the +participant plugin refuses non-HTTPS origins, so the edge is mandatory. +`nginx.prod.tls.conf.example` shows how to terminate TLS in the container +instead. + +## Required configuration + +Copy `.env.example` to `.env` and fill the "Production inventory" section. The +compose file refuses to start without: + +| Variable | Why | +| --- | --- | +| `BOOTSTRAP_SIGNING_SECRET` | HMAC key for bootstrap manifests and session/inference capabilities. Without it every bootstrap answers 503 and the backend exits at start. | +| `REDIS_PASSWORD` | `requirepass` for the session Redis. | +| `PUBLIC_ORIGIN` | The https origin participants use; becomes `CORS_ALLOWED_ORIGINS` and the website's compiled backend host. | +| `DB_NAME`, `DB_USER`, `DB_PASSWORD` | Postgres credentials (also used by the backup service). | + +Plus the provider key(s) every active agent profile's `secret_ref` names +(`OPENROUTER_API_KEY`, ...). Provider keys never leave the backend environment. + +Never reuse the development `.env`: rotate any secret that was ever committed +or shared, and keep `.env` out of images (`.dockerignore` excludes it; the +containers receive it through `env_file`). + +## First deployment + +1. `cp .env.example .env`, fill it, then validate the rendering: + `docker compose --env-file .env -f docker-compose.prod.yml config >/dev/null`. +2. `docker compose --env-file .env -f docker-compose.prod.yml up -d --build`. + `migrate` runs `migration_manager.py migrate` (initializes a fresh database + from `init.sql`, then applies every Alembic revision) and the API waits for it. +3. `docker compose -f docker-compose.prod.yml ps` — `backend` must be `healthy` + (its check is `GET /api/health`, which touches Postgres and Redis) and + `migrate` `exited (0)`. +4. Smoke the research plane through the public origin: + - `curl -fsS https:///api/health` → `{"status":"ok",...}`. + - Sign in on the website, create a study as a researcher, join it as a test + participant, then in the IDE plugin confirm the status bar reaches + **Research: active** and `docker compose logs backend | grep "telemetry batch"` + shows accepted batches. A `SIGNING_SECRET_MISSING` or `CAPABILITY_INVALID` + here means the secret is not reaching the container. +5. Confirm the first backup file appeared in `BACKUP_DIR` (the service dumps once + at start, then every `BACKUP_INTERVAL_SECONDS`). +6. Record server commit, Alembic head (`python src/database/migration/migration_manager.py current`), + plugin tag, runtime tag and checksums per `docs/MANAGED_AGENT_DEPLOYMENT.md`. + +## Upgrades + +1. Back up (`docker compose -f docker-compose.prod.yml exec backup /bin/bash /ops/backup-postgres.sh`). +2. `git pull` the release commit, `docker compose --env-file .env -f docker-compose.prod.yml build`. +3. `docker compose --env-file .env -f docker-compose.prod.yml up -d` — `migrate` + runs first; if it fails the old `backend` container keeps serving; fix or + restore before retrying. Never reset the study database during an upgrade. +4. Watch `docker compose logs -f backend` for the first heartbeats/batches. + +Schema deltas for databases created before a revision are listed in +`RELEASES.md` ("Existing data"). + +## Backups and the restore drill + +- The `backup` service writes `code4me--.dump` (pg_dump custom format) + plus a `.sha256` to `BACKUP_DIR`, prunes after `BACKUP_RETENTION_DAYS` (only + when the off-host copy succeeded), and runs `BACKUP_OFFHOST_CMD` (e.g. + `rclone copy "$1" remote:code4me-backups`) after every dump. Configure the + off-host copy before recruitment: a disk failure must not be able to take the + study with it. Set `BACKUP_DIR` to a path on a different disk than the + Postgres volume (e.g. `/var/backups/code4me`); the default `./backups` is + ignored by git and by the Docker build context, but shares the disk. The + pgvector image has no `rclone`/`aws`: mount the binary into the `backup` + service, or run the off-host copy from a host cron on `BACKUP_DIR`. +- Redis persists with AOF on `redis_data`; losing it only logs participants out + (they sign in again); nothing collected lives there. +- **Drill (do this once before the first participant, then monthly):** + ```bash + docker compose -f docker-compose.prod.yml exec backup /bin/bash /ops/backup-postgres.sh + docker compose -f docker-compose.prod.yml exec backup /bin/bash /ops/restore-postgres.sh \ + /backups/.dump --to-database code4me_drill --create + docker compose -f docker-compose.prod.yml exec db psql -U "$DB_USER" -d code4me_drill \ + -c 'SELECT count(*) FROM public.research_event;' + docker compose -f docker-compose.prod.yml exec db dropdb -U "$DB_USER" code4me_drill + ``` +- **Real restore:** stop `backend` and `celery-worker`, then + `restore-postgres.sh --yes` against `PGDATABASE`, then `up -d`. + +## Operations + +- Health: `GET /api/health` (database + Redis, 503 when either fails); + `GET /nginx-health` for the edge alone. Point your uptime monitor at both. +- Memory: the compose caps `backend` and `celery-worker` at 2 GiB. With + `CODE4ME_WEB_WORKERS=2` each worker imports the ML stack; measure RSS on the + first deploy (`docker stats`) and raise the cap before it turns into an + OOM-kill loop that looks like "healthy, then gone". +- Logs are bounded (`json-file`, 50 MB × 5 per container); the limiter's + per-request line is at DEBUG, uvicorn's access log stays on (one line per + request, no bodies or cookies). `docker compose logs backend | grep -E "telemetry batch|rejected"` + shows ingestion outcomes without payloads. +- Rate limits key on the real client address (nginx's `X-Forwarded-For`). + Research-plane paths get a 20000 requests/hour floor per client; tune with + `MAX_REQUEST_RATE_PER_HOUR_CONFIG`. +- Kill switch: the researcher UI's kill switch pauses collection; participants' + plugins keep their runtime and resume automatically when it is released. +- Session tail: events of a session that ended (idle timeout, IDE closed) are + accepted for `TELEMETRY_LATE_EVENT_GRACE_SECONDS` (default 900) after the + close; the plugin drains its spool on project close and rotates sessions on + idle without participant action. + +## Not covered here + +Alerting (silence per participant, disk usage) and the study-health page are +follow-ups (review B-11/§5.6); until they exist, check `/api/health`, disk +usage on the host, and the researcher dashboards daily during the study. diff --git a/docs/research-platform/RELEASES.md b/docs/research-platform/RELEASES.md new file mode 100644 index 0000000..f492ace --- /dev/null +++ b/docs/research-platform/RELEASES.md @@ -0,0 +1,433 @@ +# Tested release import + +> From scratch setup: [`FROM_SCRATCH_SETUP.md`](FROM_SCRATCH_SETUP.md). + +The producer owns the test results. The server verifies archive bytes and makes +passing releases usable immediately. No host approval, conformance receipt, or +“Mark tested” action is needed. `DISABLED` permanently prevents a release from +being selected for a new bootstrap; reimport cannot enable it. Existing running +sessions are not terminated by this endpoint (use the runtime kill switch). + +## Local native build + +Install the existing locked build/runtime/test dependencies as described in +`packaging/README.md`, then run **from the server repository** (the module lives +in `code4me2-server/src/research`, so `PYTHONPATH=src` only resolves there): + +```bash +cd code4me2-server + +PYTHONPATH=src python -m research.study.agents.participant_release native \ + --version 1.2.3 --platform macos-arm64 \ + --server-commit "$(git rev-parse HEAD)" --output dist/release-1.2.3 +``` + +This stamps the version, runs the runtime tests, builds with PyInstaller, archives +it, extracts that archive, and checks the extracted executable's version, +`--self-check` and ACP `initialize` response. It writes the ZIP and one +`native-macos-arm64.json`, and prints that manifest JSON on stdout. A failed +check produces no output release directory. Use a new output directory/version; +`--skip-build` is for an already built/signed native bundle, and still tests the +archived executable — with `--skip-build` the `--version` **must equal the +version already embedded in that bundle** (run without it to stamp a new +version), otherwise the producer refuses with "packaged executable version does +not match the release". Local release coverage is explicitly one native platform. +The version stamp updates `src/code4me2_agent/_build_version.py`. + +`arm64` is canonical; `aarch64` is accepted at input boundaries. The producer +requires a matching native host. Build artifacts do not imply IntelliJ UI, +permission enforcement, inference, or tool-call end-to-end acceptance. + +## GitHub release + +Run `.github/workflows/build-managed-runtime.yml` manually with a version and +`publishRelease=true`. Four native jobs build and test the binaries; with the +opt-in `sign=true` they are code-signed first and the macOS archive notarized +(see `packaging/README.md` for why participants do not need it). The publishing +job calls the same module's `merge` command, validates every ZIP, requires all +four platforms, and publishes only the combined JSON plus ZIPs, each artifact with +its exact `download_url`, under `runtime-v`. Existing tags/releases +cannot be overwritten. + +## Admin import + +Dashboard → Agent Catalogue accepts a JSON file (or pasted JSON) plus exactly +its declared ZIPs. Alternatively select “Import from release URLs”, enter a +manifest URL and one archive URL per line. Both paths perform the same checks. + +- `POST /api/research/agents/releases/import`: multipart `manifest` JSON text, + repeated `archives` file fields. +- `POST /api/research/agents/releases/import-url`: JSON `manifest_url` and + `archive_urls` string array. +- `POST /api/research/agents/releases/{release_id}/disable`: terminal disable. + +HTTPS download hosts default to `github.com`, `release-assets.githubusercontent.com` +and `objects.githubusercontent.com`. Operators can explicitly configure trusted +hosts with `RESEARCH_RELEASE_HOSTS`; every redirect is checked. Manifest downloads +are limited to 1 MiB; `RESEARCH_IMPORT_MAX_ARCHIVE_BYTES` and +`RESEARCH_IMPORT_MAX_TOTAL_BYTES` control archive/import limits. Downloads are +streamed to temporary files and removed after validation. Imported archives are +not a download CDN: participant distribution is still the published plugin/agent. + +The archive declaration is: + +```json +{ + "runtime_id": "code4me-agent", + "version": "1.2.3", + "platform": "macos", + "architecture": "arm64", + "archive": "code4me-agent-macos-arm64.zip", + "sha256": "<64 hex characters>", + "size": 12345, + "executable": "code4me2-agent", + "tests": { + "self_check": "PASS", + "acp_initialize": "PASS", + "ran_at": "2026-09-21T12:00:00Z" + } +} +``` + +Each declared archive needs its own passing results. Missing/extra/duplicate +archives, digest/size mismatches, missing or failed tests reject the import. +Every managed and BYOA row is written in one database transaction. Identical +imports are idempotent. The canonical manifest digest pins release identity; +archive SHA-256 pins distributed bytes. No extracted-file inventory is stored. + +Admin import is the trust boundary for the producer's result declaration. Hashing +bytes detects tampering; it does not independently prove tests were executed or +prove all IDE behavior. Only import output from the controlled producer/CI. + +## BYOA and plugin boundary + +The optional `agents` array carries external `framework`, explicit `version`, +`agent_command`, optional `agent_command_args`/`byoa_config`, `adapter`, and +`tests` (an array of the results above plus `os`/`arch`). External agents require +their own results; managed-runtime results never qualify Goose or Codex. Server +import never runs an uploaded command. The manifest digest binds these identities. + +The plugin is the IDE integration; the proxy observes/transports ACP; the agent +executes tasks. The plugin ships no managed agent: it installs the study's pinned +archive from its verified cache or the release's `download_url` (import accepts, +and the plugin downloads, only `download_url`s of `AISE-TUDelft/code4me2-server` +release assets); BYOA agents are installed separately. The existing signed bootstrap still pins release/version/adapter and +command/package identity. Local executable identity enforcement belongs to the +plugin and is not demonstrated by server tests. + +The plugin's recipe CLI (`scripts/participant-release.py`) is local tooling for +ZIPs with a bundled agent recipe; participant ZIPs bundle no agent and come from +the plugin workflow (see `code4me2/docs/PARTICIPANT_RELEASE.md`). It registers +nothing on a server; register releases through the website import. Plugin ZIP publication, native host permission/tool behavior and all +four CI jobs require their own verification. Server protocol restrictions on +participant-ready runtimes are unchanged. + +## Existing data + +This repository uses a consolidated fresh schema. The obsolete +`approved_hosts_json` column is removed from that schema and ORM. An existing +physical column may remain inert until the normal schema replacement process; +this change does not reset the application database. Historical releases without +platform results remain unqualified; import a newly tested manifest rather than +converting old manual approvals into passing tests. Legacy aggregate test verdicts +also stay unqualified. Existing `DISABLED`, `BLOCKED`, and `RETIRED` states remain +terminal on reads. + +Agent profiles now store an optional `system_prompt` (packaged releases only). The +column is part of the consolidated revision, so a fresh database gets it, but a +database that is already at that revision does not: every profile query fails +with an undefined-column error until you add it. Before deploying to an existing +database, run: + +```sql +ALTER TABLE public.agent_profile ADD COLUMN IF NOT EXISTS system_prompt TEXT; +``` + +The column is nullable and the previous code ignores it, so it is safe to add +while the old backend is still running. Existing profiles, their configuration +digests and frozen studies are unchanged: a prompt only enters a digest when set. + +Built-in (`code4me2-agent`) profiles can also set the commands the agent may run +(`commands_allowlist`, since replaced by `commands_denylist`; see "Blocked +commands replace the command allowlist" below), the default command timeout (`command_timeout_seconds`, +1–600 s) and the harness behaviour switches (`harness_options`); Goose and Codex +profiles refuse all three. These columns are part of the consolidated revision +too, so a database already at that revision needs them before the new backend +starts: + +```sql +ALTER TABLE public.agent_profile ADD COLUMN IF NOT EXISTS commands_allowlist_json TEXT; +ALTER TABLE public.agent_profile ADD COLUMN IF NOT EXISTS command_timeout_seconds INTEGER; +ALTER TABLE public.agent_profile ADD COLUMN IF NOT EXISTS harness_options_json TEXT; +``` + +The columns are nullable and the previous code ignores them, so they are safe to +add while the old backend is still running. NULL keeps the previous behaviour +(the server fallback allowlist, which a user's config row may replace, and the +runtime defaults). A value joins the configuration digest, the study snapshot, +`GET /api/acp/agent-config` and the managed run policy only when set, so existing +profiles, digests, frozen studies and run policies are unchanged. When a profile +sets `commands_allowlist`, a user's config-row allowlist can only narrow it. + +## Emitter-sequence uniqueness dropped (production-readiness C-01) + +Alembic revision `b7c1d2e3f4a5` (after `8a0084080b46`) drops the unique +constraint `uq_research_event_session_emitter_sequence` on `research_event` and +replaces it with the plain index `idx_research_event_session_emitter_sequence`. +An emitter whose counter restarts inside a live session (an IDE restart, a +re-launched proxy) reused `(session, emitter, sequence)` and the constraint made +ingestion reject, and the client delete, the later facts. Identity is +`event_id` + digest; overlaps are surfaced as `EVENT_SEQUENCE_GAP` coverage +diagnostics. + +`migration_manager.py migrate` applies it. For a database not tracked by +Alembic, run the equivalent by hand: + +```sql +ALTER TABLE public.research_event DROP CONSTRAINT IF EXISTS uq_research_event_session_emitter_sequence; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_research_event_session_emitter_sequence + ON public.research_event (research_session_id, emitter_id, emitter_sequence); +``` + +The `DROP CONSTRAINT` takes a brief exclusive lock; `CONCURRENTLY` keeps event +inserts flowing while the index builds (the Alembic revision builds it inside +its transaction, which blocks inserts for the few seconds a study-sized table +needs). Plan either for a quiet moment. The same +release also stores the tail of an ended session (events up to +`TELEMETRY_LATE_EVENT_GRACE_SECONDS`, default 900 s, after `closed_at`, under +the enrollment's next session capability) and closes an idle-expired context +session at bootstrap/session create instead of reusing it; neither needs a +schema change. + +## Participant inference budgets (shared provider key) + +Goose and built-in (`code4me2-agent`) study arms now spend from the study's +server-held provider key through a metered relay, and every enrollment carries +a USD budget. The consolidated revision creates the tables and columns below, so +a fresh database gets them; a database that is already at that revision does +not. Before deploying to an existing database, run: + +```sql +CREATE TABLE IF NOT EXISTS public.provider_model_price ( + connection_id UUID NOT NULL REFERENCES public.provider_connection(connection_id) ON DELETE CASCADE, + model TEXT NOT NULL, + input_usd_per_million NUMERIC(14,6) NOT NULL CHECK (input_usd_per_million >= 0), + output_usd_per_million NUMERIC(14,6) NOT NULL CHECK (output_usd_per_million >= 0), + cached_input_usd_per_million NUMERIC(14,6) NULL CHECK (cached_input_usd_per_million IS NULL OR cached_input_usd_per_million >= 0), + updated_at TIMESTAMPTZ NOT NULL, + updated_by TEXT NULL, + PRIMARY KEY (connection_id, model) +); +CREATE INDEX IF NOT EXISTS idx_provider_model_price_connection ON public.provider_model_price (connection_id); + +ALTER TABLE public.study + ADD COLUMN IF NOT EXISTS inference_budget_default_micro_usd BIGINT NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS inference_budget_warning_fraction NUMERIC(4,3) NOT NULL DEFAULT 0.800, + ADD COLUMN IF NOT EXISTS inference_budget_updated_at TIMESTAMPTZ NULL, + ADD COLUMN IF NOT EXISTS inference_budget_updated_by TEXT NULL; + +CREATE TABLE IF NOT EXISTS public.enrollment_inference_balance ( + enrollment_id UUID PRIMARY KEY REFERENCES public.research_enrollment(enrollment_id) ON DELETE CASCADE, + study_id UUID NOT NULL REFERENCES public.study(study_id) ON DELETE CASCADE, + unit TEXT NOT NULL DEFAULT 'micro_usd', + limit_micro_usd BIGINT NOT NULL CHECK (limit_micro_usd >= 0), + settled_micro_usd BIGINT NOT NULL DEFAULT 0 CHECK (settled_micro_usd >= 0), + reserved_micro_usd BIGINT NOT NULL DEFAULT 0 CHECK (reserved_micro_usd >= 0), + settled_prompt_tokens BIGINT NOT NULL DEFAULT 0, + settled_completion_tokens BIGINT NOT NULL DEFAULT 0, + call_count INTEGER NOT NULL DEFAULT 0, + refused_count INTEGER NOT NULL DEFAULT 0, + last_call_at TIMESTAMPTZ NULL, + limit_source TEXT NOT NULL DEFAULT 'STUDY_DEFAULT', + exhausted_at TIMESTAMPTZ NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_enrollment_inference_balance_study_id ON public.enrollment_inference_balance (study_id); + +CREATE TABLE IF NOT EXISTS public.inference_reservation ( + reservation_id UUID PRIMARY KEY, + enrollment_id UUID NOT NULL REFERENCES public.enrollment_inference_balance(enrollment_id) ON DELETE CASCADE, + study_id UUID NOT NULL, + connection_id UUID NULL REFERENCES public.provider_connection(connection_id) ON DELETE SET NULL, + model TEXT NOT NULL, + entry_point TEXT NOT NULL, + request_id TEXT NOT NULL UNIQUE, + research_session_id UUID NULL, + state TEXT NOT NULL CHECK (state IN ('RESERVED', 'SETTLED', 'FORFEITED', 'VOIDED', 'EXPIRED')), + hold_micro_usd BIGINT NOT NULL CHECK (hold_micro_usd >= 0), + estimated_prompt_tokens INTEGER NOT NULL, + output_cap_tokens INTEGER NOT NULL, + charged_micro_usd BIGINT NULL CHECK (charged_micro_usd IS NULL OR charged_micro_usd >= 0), + prompt_tokens INTEGER NULL, + completion_tokens INTEGER NULL, + cached_prompt_tokens INTEGER NULL, + usage_source TEXT NULL, + resolution_reason TEXT NULL, + upstream_status INTEGER NULL, + finish_reason TEXT NULL, + reserved_at TIMESTAMPTZ NOT NULL, + deadline_at TIMESTAMPTZ NOT NULL, + resolved_at TIMESTAMPTZ NULL +); +CREATE INDEX IF NOT EXISTS idx_inference_reservation_enrollment_state ON public.inference_reservation (enrollment_id, state); +CREATE INDEX IF NOT EXISTS idx_inference_reservation_open_deadline ON public.inference_reservation (deadline_at) WHERE state = 'RESERVED'; +CREATE INDEX IF NOT EXISTS idx_inference_reservation_study_reserved_at ON public.inference_reservation (study_id, reserved_at); + +CREATE TABLE IF NOT EXISTS public.inference_budget_adjustment ( + adjustment_id UUID PRIMARY KEY, + enrollment_id UUID NOT NULL REFERENCES public.enrollment_inference_balance(enrollment_id) ON DELETE CASCADE, + study_id UUID NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('TOP_UP', 'SET_LIMIT', 'APPLY_DEFAULT', 'BACKFILL')), + delta_micro_usd BIGINT NOT NULL, + limit_before_micro_usd BIGINT NOT NULL, + limit_after_micro_usd BIGINT NOT NULL, + in_flight_micro_usd BIGINT NOT NULL DEFAULT 0, + reason TEXT NULL, + actor TEXT NULL, + idempotency_key TEXT NULL, + request_digest TEXT NULL, + occurred_at TIMESTAMPTZ NOT NULL, + UNIQUE (study_id, idempotency_key) +); +CREATE INDEX IF NOT EXISTS idx_inference_budget_adjustment_enrollment ON public.inference_budget_adjustment (enrollment_id, occurred_at); +CREATE INDEX IF NOT EXISTS idx_inference_budget_adjustment_study ON public.inference_budget_adjustment (study_id, occurred_at); + +-- Give every existing enrollment a balance row at its study's default. +INSERT INTO public.enrollment_inference_balance (enrollment_id, study_id, limit_micro_usd, limit_source, created_at, updated_at) +SELECT e.enrollment_id, e.study_id, s.inference_budget_default_micro_usd, 'BACKFILL', now(), now() +FROM public.research_enrollment e +JOIN public.study s ON s.study_id = e.study_id +ON CONFLICT (enrollment_id) DO NOTHING; +``` + +This is fail-closed on purpose. A study's default budget is 0 until a +researcher sets it (study Settings → Participant budgets, then "Apply new +default" for the participants already enrolled), and a model without a price +row on its provider connection refuses every call with `503 price_missing`. +Until both are configured, Goose and built-in arms receive `402 quota_exhausted` +or `503 price_missing` and spend nothing. Codex arms are unaffected: they sign in +with ChatGPT and are not metered. + +## Self-service privacy controls (GDPR) + +Every account can now opt out of data collection and erase what Code4Me +collected about it, from the website ("Privacy & data") or the plugin +(`GET /api/user/privacy`, `PUT /api/user/privacy/collection`, +`POST /api/user/privacy/erase`). The opt-out is stored in a new account column +that only these endpoints write: + +```sql +ALTER TABLE public."user" ADD COLUMN IF NOT EXISTS data_collection_opted_out_at TIMESTAMPTZ NULL; +``` + +The column is part of the consolidated revision, so a fresh database gets it; a +database already at that revision needs the statement above before the new +backend starts, or every account query fails with an undefined-column error. It +is nullable and the previous code ignores it, so it is safe to add while the old +backend is still running. + +Erasing an account deletes its context and telemetry rows, and each delete +checks `meta_query` for references. `init.sql` now indexes those foreign keys; an +existing database should add the indexes (without them an erase scans +`meta_query` once per deleted row, which takes minutes on a large table): + +```sql +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_meta_query_context_id ON public.meta_query (context_id) WHERE context_id IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_meta_query_contextual_telemetry_id ON public.meta_query (contextual_telemetry_id) WHERE contextual_telemetry_id IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_meta_query_behavioral_telemetry_id ON public.meta_query (behavioral_telemetry_id) WHERE behavioral_telemetry_id IS NOT NULL; +``` + +A concurrent build that fails leaves an invalid index under that name, which a +re-run then skips. Check with +`SELECT indexrelid::regclass FROM pg_index WHERE NOT indisvalid;` and drop and +re-create any of the three it lists. + +While an account is opted out the server stores nothing about its completion, +chat and feedback requests, denies agent content and agent memory, never flushes +its project context, and refuses study joins; opting out withdraws an active +enrollment (new status `WITHDRAWN`). Erasing deletes the account's classic, agent and +research data in one transaction and leaves one content-free deletion-ledger +record per erased enrollment. `DELETE /api/user/delete` now always erases the +data (`delete_data` is ignored), revokes the account's tokens, and refuses with +`409` an account that owns research studies or agent profiles. + +## Study consent versions and assignment policy + +Studies can carry their own consent document and tick-box statements, and each +enrollment records which consent view it accepted (`consent_digest`) together with +that view and the participant's answers (`consent_snapshot_json`). Alembic +revision `c5e8f1a2d3b4` (after `b7c1d2e3f4a5`) adds the two columns, so the +deployment step `python src/database/migration/migration_manager.py migrate` +applies it (the new backend needs the columns before it starts). It is +idempotent; run by hand it is: + +```sql +ALTER TABLE public.research_enrollment + ADD COLUMN IF NOT EXISTS consent_digest VARCHAR NULL, + ADD COLUMN IF NOT EXISTS consent_snapshot_json JSONB NULL; +``` + +Both are nullable and the previous code ignores them, so they are safe to add +while the old backend is still running. Enrollments created earlier keep null +values: their consent version was never recorded. + +Studies created from this release on freeze an `assignment` block in their +configuration (`DETERMINISTIC_HASH` salted-hash draw, plus the opt-in manual +override); existing studies keep the `RANDOM_EQUAL` draw. This needs no schema +change. See `docs/RESEARCH_ANALYTICS_SCOPE.md`. + +## Blocked commands replace the command allowlist + +Built-in (`code4me2-agent`) profiles no longer list the programs the agent may +run. They list the programs it may **not** run (`commands_denylist`, "Blocked +commands" in the profile editor, at most 64 names); every other program installed +on the participant's machine can run. Under the per-step approval policy the +participant still approves each command; under auto-approval nothing asks. An +empty list blocks nothing. + +Alembic revision `d41e7c9a2b6f` (after `c5e8f1a2d3b4`) adds the column, so +`python src/database/migration/migration_manager.py migrate` applies it (the new +backend needs it before it starts). By hand: + +```sql +ALTER TABLE public.agent_profile ADD COLUMN IF NOT EXISTS commands_denylist_json TEXT; +``` + +It is nullable and the previous code ignores it, so it is safe to add while the +old backend is still running. The retired `commands_allowlist_json` column stays +in the table, unused, so the previous backend can still be rolled back to. +Profile templates saved with an allowlist show an empty "Blocked commands" field; +studies frozen earlier keep their snapshot. + +The packaged agent enforces the list, so studies need a runtime release built +from this code (the release after `runtime-v0.0.6`). Releases up to and including +`runtime-v0.0.6` ignore `commands_denylist` and keep their old behaviour: they +run only the commands in `commands_allowlist`, which the server still sends, +computed as before (an allowlist frozen in an older study, else a user's +config-row allowlist, else `pwd, ls, cat, grep, rg`) minus any blocked command. +A newer runtime refuses a run policy without `commands_denylist` (its log says +the server is older than the runtime), so deploy the server first, then import +the new runtime release and pin profiles to it. + +A name matches the program a command starts: case-insensitive, ignoring any +directory, trailing dots and spaces, and a `.exe`/`.cmd`/`.bat`/`.com` suffix +(`git` also blocks `/usr/bin/git` and `git.exe`). When the command is a shell +(for example `bash`, `sh`, `zsh`, `tcsh`, `cmd /c`, `powershell -Command`, `wsl`), +a launcher (for example `env`, `xargs`, `sudo`, `timeout`, `script`, `npx`) or a +package manager (for example `uv`, `poetry`, `conda`, `npm`, `pnpm`, `yarn`, +`mise`), every plainly written word of its arguments is checked too (`@` splits +words, so neither a version suffix such as `npm@10` nor `git@github.com` hides a +name), so `bash -c "git push"` and `npm x -c "git push"` are refused. Package managers have +many ways to run a program, so all their arguments count: installing a package +named like a blocked program is refused as well. The lists in `agents/tools.py` are not +exhaustive. A program started by a script, an interpreter (`python -c`), a build +tool (`make`, `npm run` scripts) or `find -exec`, and a name disguised with +escapes, is not visible to the check: block those as well if that matters, or +rely on per-step approval. A verify command may not use a blocked program; this +is refused when a profile is saved or frozen, while a running study whose verify +command a config row (or a later rule change) blocks has the runtime skip it. A +user's config row can add bare names with `{"agent": {"commands_denylist": +[...]}}`; a malformed one fails closed (a managed run is refused; a legacy caller +loses `run_command`, as it does whenever the blocked commands cannot be read). +Its legacy `commands_allowlist` only affects runtimes up to 0.0.6. diff --git a/e2e/.gitignore b/e2e/.gitignore new file mode 100644 index 0000000..b90d1ca --- /dev/null +++ b/e2e/.gitignore @@ -0,0 +1,8 @@ +runs/ +.cache/ +.venv/ +.browser-venv/ +.ruff_cache/ +__pycache__/ +*.pyc +.DS_Store diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000..f76b111 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,423 @@ +# Code4Me end-to-end tests + +This harness is version-controlled in the server repository at +`code4me2-server/e2e/`. It drives both checkouts, so it expects the workspace +layout: + +``` +/ + code4me2/ # IntelliJ plugin + code4me2-server/ # backend + this harness +``` + +From the workspace root: + +```bash +./code4me2-server/e2e/test +``` + +or from this directory: + +```bash +./test +``` + +`CODE4ME_E2E_WORKSPACE` overrides the workspace lookup (CI checks the two +repositories out side by side and sets it when the layout differs). + +One command runs everything, and nobody has to prepare the host by hand first. +The gate checks each layer's prerequisites and provisions what the harness can +own: it starts Colima when the Docker daemon is down, builds the backend image +when it is missing or its `requirements.txt` changed, creates the Playwright +venv and its Chromium, installs website dependencies, and builds the plugin's +vendored Codex ACP adapter. It then starts a disposable backend, builds the +current plugin and native runtimes, opens an isolated IntelliJ sandbox, drives +the UI, exercises the registered ACP proxy, and checks the resulting telemetry in +PostgreSQL. It runs the live Kotlin fixture and the HTTP workflow through +enrollment revocation, real Goose and Codex agents over ACP, and the website +suite. No manual IDE startup, account creation, study setup, or model API key is +needed. + +On success the default command closes its IDE and removes its Compose stack and +volume. On failure it closes the IDE and keeps the stack and run artifacts for +diagnosis. A nonzero exit code means the gate did not pass. + +## For coding agents + +- Run the whole gate with `./code4me2-server/e2e/test --json` from the + workspace root. It prints progress on stderr and a JSON summary on stdout: + `ok`, every step, `layers` (PASS / FAIL / BLOCKED / NOT_ATTEMPTED with a + reason) and `prerequisites` (each check with its remediation). +- **Run it outside any command sandbox, in a logged-in desktop session.** The + IDE layer opens a real IntelliJ window. Started from a sandboxed shell (Claude + Code's Bash sandbox, Codex's seatbelt), IntelliJ hangs in AWT/Metal start-up + and robot-server never answers; the harness then fails at `ide_startup` after + its timeout. In Claude Code, run the command with the sandbox disabled. +- `python3 -m code4me_e2e setup --check` (from `code4me2-server/e2e/`) reports + every prerequisite without changing anything; `setup` provisions them without + running tests. Both take `--layer` and `--json`. +- A layer that cannot get its prerequisites is BLOCKED with the exact + remediation, and the gate fails. Nothing is skipped silently. The other, + independent layers still run: `agents` and `browser` run even when the IDE + layer fails. +- With warm caches a full run takes about 3 minutes (2 min 52 s on 2026-09-25: + IDE layer 51 s, Kotlin fixture 80 s, agents 6 s, browser 23 s). The first run + takes much longer: Gradle downloads the IDE, PyInstaller builds the agent and + proxy, and a missing backend image downloads several GB. Use `--layer` for a + narrower check (table below). +- Only one harness command runs at a time (workspace lock). Do not run other + Gradle builds in `code4me2/` while the gate runs. The gate stops sandbox IDEs + left behind by an earlier, interrupted harness run. +- Give the command a long timeout or run it in the background: provisioning on a + fresh host (image build, IDE download, native builds) can take far longer than + a warm run. An interrupted gate (Ctrl-C or SIGTERM, e.g. a tool timeout) stops + the builds and the IDE it started; SIGKILL cannot be cleaned up. + +## Choose the checks + +| Command | Coverage | +| --- | --- | +| `./test` | every layer below: real IDE + registered ACP proxy, Kotlin fixture, HTTP workflow, real agents, website | +| `./test --layer backend` | 17 HTTP workflow steps (imports the producer-built agent release; no IDE) | +| `./test --layer plugin` | Kotlin IntelliJ fixture + HTTP workflow, no IDE UI | +| `./test --layer agents` | real Goose and Codex over ACP against a local provider; no Docker | +| `./test --layer browser` | `browser/` suite: 40 headless Chromium checks + persisted read models | +| `./test --json` | Same gate, with a JSON summary on stdout | +| `./test --keep-stack` | Preserve the disposable backend after success | +| `./test --no-setup` | Check prerequisites but provision nothing (missing ones block their layers) | + +`all` is the default and runs the layers in this order: `ui`, `plugin`, +`backend`, `agents`, `browser`. `plugin` and `backend` depend on the IDE layer +and are NOT_ATTEMPTED when an earlier layer failed. `agents` and `browser` are +independent and always attempted. The report records every layer +(`report.json` → `layers`). + +The IDE layer uses the real server, plugin, packaged agent and packaged proxy. +Only the external model provider is replaced by a deterministic local HTTP +provider. The UI test drives eight steps: plugin loading, settings navigation, +sign-in, automatic enrollment discovery, the status surface, ACP registration, +agent preparation and one model response. The harness then checks the entry the +IDE registered in its ACP registry: +- the proxy and the wrapped agent binary are executable files; +- their sha256 and `--version` are recorded; +- the declared `--agent-digest` matches the agent binary; +- the one-time spool capability is present. + +It then acts as an ACP client with that exact command, arguments and managed-auth +environment, and requires: +- the provider's deterministic token in the answer; +- exactly one new provider request for that first prompt; +- canonical telemetry scoped to this launch, persisted in the database: + `interaction.started` ×2, `agent.message.started` ×2 and + `agent.message.completed` ×1 from this proxy emitter, plus IDE events from this + run's research session. + +The Kotlin fixture independently tests the plugin's login, discovery, signed +bootstrap validation and durable spool upload. + +The browser suite covers: +- study creation, consent and enrollment handoff; +- metadata editing and locking; +- the participants, dashboard and analytics tabs, and the participant's My studies page; +- stopping and cloning a study; +- authorization controls. + +It must run its exact 40-check inventory once each. It then re-reads the real +API read models: +- edited metadata; +- stopped study retention; +- clone identity and profile selection; +- enrollment and assignment coverage. + +The browser layer seeds its own accounts, profile and study under +`runs//browser-state/`, so it never disturbs the IDE layers' state. + +This does not automate JetBrains AI Assistant's chat panel or JetBrains account +sign-in. It tests the registered ACP integration through its public protocol. +The deterministic provider does not test model quality, external provider +availability, or every agent/tool combination. + +## Prerequisites: provisioned vs. one-time + +`./test` and `setup` check these per layer, in this order: + +| Check | Layers | Provisioned by the harness | Otherwise BLOCKED with | +| --- | --- | --- | --- | +| `docker` | ui, plugin, backend, browser | `colima start [profile]` when the daemon is down and the active docker context is a Colima one (never creates a VM or switches contexts) | start Docker Desktop or Colima | +| `backend_image` | same | `docker build -f Dockerfile.cpu` when the image is missing, or its `/app/requirements.txt` differs from the checkout (harness builds carry an `org.code4me.e2e.deps` label) | the build log path | +| `stack_ports` | same | — (ports held by this harness's own compose project are fine) | the owning process; `--set stack._port=N` | +| `native_python` | ui, plugin, backend, browser | `e2e/.venv`, pinned like the release CI (`packaging/requirements-{runtime,build}.lock` + the editable server), when neither `code4me2-server/.venv` nor an existing `e2e/.venv` imports the build inputs. The first interpreter that does is used | an explicit `CODE4ME_E2E_PYTHON` that cannot import them is BLOCKED, never replaced | +| `java` | ui, plugin | — | install a JDK 17+ (Gradle provisions the plugin's toolchain and IDE) | +| `gui_session` | ui | — | run from a logged-in desktop session (Linux: `DISPLAY`/Xvfb) | +| `stale_ides` | ui | stops sandbox IDEs left by an interrupted harness run (only processes whose home is under `e2e/runs/`) | — | +| `goose` | agents | — | install Goose once or set `CODE4ME_E2E_GOOSE_EXECUTABLE` | +| `node` | agents, browser | — | install Node.js 18+ (22 is verified) | +| `codex_acp` | agents | `npm ci` + `npm run build` of `code4me2/dev/codex-acp-proxy/codex-acp` (gitignored outputs) and a launcher at `e2e/.cache/agents/bin/codex-acp` | the build log; or `CODE4ME_E2E_CODEX_EXECUTABLE` | +| `website_deps` | browser | `npm ci` in `src/website` when `node_modules` is absent | the install log | +| `browser_python` | browser | `e2e/.browser-venv` with Playwright 1.63.0 and its Chromium (browsers go to Playwright's shared per-user cache) | the venv command (an explicit `CODE4ME_E2E_BROWSER_PYTHON` is never replaced) | + +Provisioning logs go to `e2e/.cache/setup/`. The first backend image build +downloads several GB. The image tag `code4me2-server-backend:latest` is the one +`docker-compose.dev-arm.yaml` uses: the harness rebuilds it only when its +`requirements.txt` differs from the checkout, which is the image the dev stack +needs too. Set `--set stack.image=` to keep a separate e2e image. One-time host tools the harness never installs: Docker +(or Colima), a JDK, Node.js, Python 3.10+ and Goose. The harness itself uses only +the standard library; the harness unit tests additionally import the server's +`research.*` package from the server venv. + +Each test recreates its backend with current server source mounted read-only and +applies Alembic migrations; ordinary source edits need no image rebuild. The +harness uses synthetic `backend.env`, never the developer's `.env`. + +The browser runner builds current website source into its run directory and +pins login and research API traffic to its local proxy. It needs no manually +started web server or previously seeded account. + +## Real agents: Goose and Codex + +A `goose` or `codex` study arm is participant-installed (`BYOA_EXTERNAL`), so +the harness only launches an agent it can identify. The `agents` layer and the +standalone probe use no Compose stack and no IDE. `agent-probe` is the only +command exempt from the workspace lock. + +```bash +cd code4me2-server/e2e +# what the agents layer runs: a real ACP turn against a local deterministic provider +python3 -m code4me_e2e agent-probe --framework goose --local-provider --json +python3 -m code4me_e2e agent-probe --framework codex --local-provider --json +# identify only: path, size, sha256 and --version; no session and no model turn +python3 -m code4me_e2e agent-probe --framework goose --detect-only --json +``` + +Codex runs through the plugin's vendored ACP adapter +(`code4me2/dev/codex-acp-proxy/codex-acp`, the adapter the plugin ships). +`setup --layer agents` (and the gate) builds it; `agent-probe`, which runs +outside the workspace lock, never builds it and reports `missing_binary` until +it is built instead of falling back to another `codex-acp` on `PATH`. Goose is +the host installation. + +The executable is resolved in this order: +1. `--executable` or the `agent.executable` setting; +2. `CODE4ME_E2E_GOOSE_EXECUTABLE` / `CODE4ME_E2E_CODEX_EXECUTABLE`; +3. a release-style `agent.agent_command` / `agent.agent_package`; +4. for Codex, the vendored adapter (and nothing else); +5. for Goose, `PATH`, then the plugin's known per-user install locations. + +Each probe records the absolute path, size, sha256 and the first `--version` +line. It runs under an isolated agent home with ambient provider credentials +removed, and exercises `initialize` → `session/new` → one `session/prompt` over +stdio with fail-closed host responses. + +With `--local-provider` (and in the `agents` layer), the agent talks to a +loopback provider that answers with a unique token: +- Goose uses Chat Completions; +- the Codex adapter uses the Responses API through `CODEX_PROXY_URL`. + +A pass needs both a fresh request at that provider and the token in the answer. + +A missing binary, missing authentication, exhausted quota or protocol failure is +a typed `BLOCKED` reason (`missing_binary`, `missing_auth`, `quota`, +`protocol`) with exit code 1. `BLOCKED` is never a pass. `--home DIR` overrides +the isolated agent home, which must be empty. `--run-dir DIR` keeps the probe +transcript and stderr log. Without `--local-provider`, the probe uses the +agent's own provider configuration, which the isolated home does not have, so +expect `missing_auth` at the prompt. + +`codex-acp` is launched without a subcommand. The bare `codex` CLI does not +speak ACP on stdio and is refused with a `protocol` block rather than left +hanging; pass `agent.agent_command_args` to use another adapter. + +Workflow scenarios always use the packaged `code4me2-agent`. Goose and Codex +releases can only be imported through the release manifest, and their arms are +not run by the managed relay, so `--set agent.framework_version=goose|codex` +is refused with `FRAMEWORK_NOT_IN_WORKFLOW`. + +For Goose the probe uses the *gateway shape* a study arm receives: `OPENAI_HOST` +is the provider origin, `OPENAI_BASE_PATH` is the research gateway path +(`api/research/inference/v1/chat/completions`), `OPENAI_API_KEY` is a random +bearer the provider must see back, `GOOSE_PROVIDER=openai` and +`GOOSE_PATH_ROOT` isolates Goose's own state while the isolated home carries a +deliberately *poisoned* `.config/goose/config.yaml` (another provider, a dead +host). A pass proves the pinned Goose honours the env-driven gateway +configuration over its own config. `--quota-exhausted` makes the provider answer +`402 quota_exhausted` (the research gateway's refusal) and passes only when the +probe is `BLOCKED` with reason `quota` after no more than Goose's two calls of a +normal turn (the reply and the session-description call) — no retry storm. +Goose 1.51 reports that refusal over ACP as `credits_exhausted` with its own +generic "add more credits" wording; the study-specific explanation reaches the +participant through the plugin's status banner. + +### Unverified boundaries + +- **Real provider authentication** is not exercised (the provider is a local + fixture); the budget refusal *shape* is, through `--quota-exhausted`. A passing + probe is not evidence that a paid provider works for a participant. +- **Codex model binding:** the adapter's `CODEX_MODEL` is not applied when it + talks to a custom gateway. The local provider sees Codex's default model name, + so the probe does not assert the model. +- **JetBrains-hosted chat** (AI Assistant's chat panel and JetBrains account + sign-in) is not automated. The UI layer drives setup and registration; the + registered ACP proxy and the real agents are exercised over stdio instead. +- Goose/Codex builds and tool catalogues vary by installation. The probe records + the exact executable it used (`identity` in `--json`), but cannot certify an + unmeasured build or a tool combination it did not observe. + +## Isolation and repeatability + +- **Compose project and ports.** The default project is `code4me-e2e`, with: + - backend port **28008**; + - database port **25432**; + - Redis port **26379**; + - provider stub port **28999**. + + Database, Redis and backend ports bind to loopback. Project-name overrides must + start with `code4me-e2e-`, so destructive cleanup cannot select the developer's + Compose project. Every fresh run, and every layer state directory, creates + unique account emails. + + The stub listens on loopback too (`stack.stub_host`). The backend container + calls it through `host.docker.internal`, which Docker Desktop and Colima + forward to the host's loopback. A Linux Docker engine maps that name to the + bridge gateway instead, so run with `--set stack.stub_host=0.0.0.0` there. +- **IDE runs.** Each IDE run has a private home under `runs//ide-home/`, + including its ACP registry, credentials, project and caches. Credentials are + kept in memory (PasswordSafe `MEMORY_ONLY`), so the sandbox never reads or + prompts for the developer's macOS keychain. robot-server uses an ephemeral + port. Cleanup targets only that run's processes. The harness never edits the + developer's `~/.jetbrains/acp.json`. +- **Native build cache.** Native builds are cached under + `code4me2-server/e2e/.cache/`, keyed by source content, dependency lock and + packaging inputs. The release producer + (`research.study.agents.participant_release native`) writes the manifest the + backend imports and the archive the UI layer seeds into the IDE's agent cache + (the plugin bundles no agent). Release + versions and checked-in runtime resources are not stamped. After changing the + host Python environment, remove the harness-owned `.cache/agent/` to force a + native rebuild. +- **One command at a time.** A workspace lock prevents simultaneous harness + commands. Do not run unrelated Gradle tests or builds in `code4me2/` while the + gate runs: Gradle sandbox staging and JUnit output directories are shared. +- **What fails the gate:** + - missing test output; + - missing UI steps; + - skipped tests; + - failing Gradle commands; + - missing or duplicated browser checks; + - blocked prerequisites. + +## Reports and debugging + +Each run prints its artifact directory, +`code4me2-server/e2e/runs/-/`: + +- `report.json`: step status, timing, assertions and failure hints, plus + `layers` and `prerequisites`. +- `state.json`: resumable synthetic accounts, IDs and credentials (mode 0600). +- `http.jsonl`: HTTP exchanges with secret-shaped fields redacted. +- `ui-test-results.json`, `ui-test.log`, `ui-ide.log`: actual IDE assertions and + Gradle/IDE output. +- `acp-runtime.log`: managed native agent/proxy diagnostics. +- `plugin-test.log`: the live Kotlin fixture result. +- `native-agents//`: real-agent probe transcripts and stderr logs. +- `browser-results.json`, `browser-test.log`, `browser-build.log`, + `browser-state/`: browser checks and the layer's own seeded state. +- `logs/backend.log`: backend diagnostics captured on failure. + +Run directories are private and ignored by Git. Keep `state.json`, IDE homes and +raw process logs local, because they can contain test credentials or synthetic +conversation content. Share a reviewed `report.json` when reporting a failure. + +A synthetic admin is promoted through SQL because the API has no admin-creation +endpoint; this is recorded as `NO_ADMIN_GRANT_ENDPOINT`, not a skipped check. +The imported agent release is QUALIFIED because the producer's own self-check +and ACP `initialize` results passed. That is test setup, not production +conformance certification. + +For focused debugging, run the module from this directory: + +```bash +cd code4me2-server/e2e +python3 -m code4me_e2e setup --check +python3 -m code4me_e2e doctor --json +python3 -m code4me_e2e ui-test --keep-ide --keep-stack +python3 -m code4me_e2e plugin-test --keep-stack +python3 -m code4me_e2e run --from bootstrap --run-dir runs/ --keep-stack +python3 -m code4me_e2e step telemetry --state runs//state.json --json +python3 -m code4me_e2e stack status +python3 -m code4me_e2e stack down +``` + +`ui-test`, `plugin-test`, `run` and `step` do not provision; run `setup` first +when you use them on a fresh host. The exit code of `run` covers only the steps +that invocation ran; the report (and its `ok`) keeps earlier results of the run. `--keep-ide` is an explicit debugging escape +hatch: it leaves that sandbox open, and the next gate stops it as a leftover. + +Resuming a run: +- Resumed HTTP layers authenticate again, because IDE/fixture login invalidates + older cookies. +- Resume only against the same retained stack and a compatible scenario. +- Use a fresh run after runtime/profile changes or enrollment revocation. +- Stack teardown deletes the disposable database, so saved IDs cannot resume + against a newly created one. + +Scenario overrides work on every test layer: + +```bash +./test --scenario scenario.example.json --keep-stack +./test --layer backend --set stack.backend_port=28009 +``` + +The example lists supported settings. Without an explicit `base_url`, it follows +`stack.backend_port`. Mutating commands require this disposable loopback origin; +`doctor --base-url URL` is the read-only exception. The historical step ID +`plugin_join` now exercises the **website** consent API; IntelliJ discovers an +existing enrollment automatically. + +## Harness regression tests + +These need no Docker, IDE, real agent or network provider: + +```bash +cd code4me2-server +PYTHONPATH=e2e .venv/bin/python -m unittest discover -s e2e/tests -v +``` + +What the tests cover: +- `tests/test_prereqs.py`: every check and provisioning decision, with patched + commands; nothing is started, built or installed. +- `tests/test_real_agents.py`: + - discovery and identity; + - the typed `BLOCKED` reasons and the Codex ACP guard; + - scoped ACP telemetry, checked against the server's own normalizer; + - stub receipts and registered-entry identity. + + It uses temporary directories and recorded fake transcripts only. +- `tests/test_suite_coverage.py`: layer gating, the browser inventory, the + persisted re-reads and browser state isolation. + +The normalizer contract check imports the server's editable `research.*` +package. With an interpreter that lacks it, the check is reported as +**skipped** with an explicit reason, never as a pass. The venv command above +runs it. The three `test_classic_provider_e2e` tests are opt-in live checks +against paid OpenRouter (`E2E_CLASSIC_PROVIDER=1`) and are skipped otherwise. + +## Continuous integration + +`.github/workflows/e2e.yml` runs the harness unit tests on every push or pull +request that touches the harness or the backend it drives (`src/` without the +website, `packaging/`, `Dockerfile.cpu`, `requirements.txt`, `pyproject.toml`, +`alembic.ini`). The `backend` layer runs for pushes to `test_draft` and `main`, +for pull requests and on demand; the other layers run on demand only. +- The runner's Docker engine is Linux, so the workflow passes + `--set stack.stub_host=0.0.0.0` (see *Compose project and ports* under + Isolation and repeatability). +- Both repositories must be checked out side by side; the workflow sets + `CODE4ME_E2E_WORKSPACE` to the directory holding them. +- Missing prerequisites are provisioned the same way as locally, including + `e2e/.venv` for the native builds. +- The IDE/UI layer needs a graphical runner and is not part of the default CI + job. + +The browser scripts are vendored at `browser/` next to the harness. A developer +checkout may still keep them at `/task07-browser/`; the harness uses +that location as a fallback. diff --git a/e2e/backend.env b/e2e/backend.env new file mode 100644 index 0000000..8993732 --- /dev/null +++ b/e2e/backend.env @@ -0,0 +1,28 @@ +# Synthetic configuration only. No developer credentials or paid providers. +SERVER_VERSION_ID=1 +AUTHENTICATION_TOKEN_EXPIRES_IN_SECONDS=86400 +SESSION_TOKEN_EXPIRES_IN_SECONDS=86400 +EMAIL_VERIFICATION_TOKEN_EXPIRES_IN_SECONDS=86400 +RESET_PASSWORD_TOKEN_EXPIRES_IN_SECONDS=3600 +TOKEN_HOOK_ACTIVATION_IN_SECONDS=60 +DEFAULT_MAX_REQUEST_RATE_PER_HOUR=100000 +MAX_REQUEST_RATE_PER_HOUR_CONFIG={} +DB_POOL_SIZE=5 +DB_MAX_OVERFLOW=10 +DB_POOL_TIMEOUT=30 +DB_POOL_RECYCLE=3600 +PGADMIN_HOST=localhost +PGADMIN_PORT=5050 +PGADMIN_DEFAULT_EMAIL=e2e@example.com +PGADMIN_DEFAULT_PASSWORD=e2e-only +WEBSITE_HOST=localhost +WEBSITE_PORT=3000 +REACT_APP_GOOGLE_CLIENT_ID=e2e-unused +EMAIL_HOST=localhost +EMAIL_PORT=1025 +EMAIL_USERNAME=e2e-unused +EMAIL_PASSWORD=e2e-unused +EMAIL_USE_TLS=false +EMAIL_FROM=e2e@example.com +VERIFICATION_URL=http://localhost:3000/verify +RESET_PASSWORD_URL=http://localhost:3000/reset diff --git a/e2e/browser/admin_flow.js b/e2e/browser/admin_flow.js new file mode 100644 index 0000000..c3a47c8 --- /dev/null +++ b/e2e/browser/admin_flow.js @@ -0,0 +1,474 @@ +// End-to-end product-flow driver for the administrator panel slice. +// +// It exercises the three new admin Dashboard views (accounts, provider +// connections, agent catalogue) in a real Chromium, then drives the existing +// researcher profile, study-creation, and participant-join pages, and finally +// checks the participant's bootstrap eligibility over HTTP. +// +// It reuses the repository's Playwright harness (same Chrome/Chromium channel +// discovery as e2e/browser/scenarios.py) and the running dev stack; it adds no +// runtime dependency to the website. The Playwright module is resolved from an +// existing install (env CODE4ME_PLAYWRIGHT_MODULE, a normal `require`, or the +// npm `npx` cache) and launched through the installed Google Chrome channel. +// +// Env: +// CODE4ME_FLOW_UI_URL (default http://localhost:3000) +// CODE4ME_FLOW_API_URL (default http://localhost:8008) +// CODE4ME_FLOW_ADMIN_EMAIL (default admin@example.com) +// CODE4ME_FLOW_ADMIN_PASSWORD (default Code4me-dev1) +// CODE4ME_FLOW_MANIFEST (required: producer manifest native-.json with its ZIP beside it) +// CODE4ME_FLOW_SECRET_REF (default OPENAI_API_KEY; the *name* only) +// CODE4ME_FLOW_CHANNEL (default chrome; set to "" for bundled chromium) +// CODE4ME_FLOW_REPORT (optional JSON report path) + +"use strict"; + +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { randomUUID } = require("crypto"); + +const UI = process.env.CODE4ME_FLOW_UI_URL || "http://localhost:3000"; +const API = process.env.CODE4ME_FLOW_API_URL || "http://localhost:8008"; +const ADMIN = { + email: process.env.CODE4ME_FLOW_ADMIN_EMAIL || "admin@example.com", + password: process.env.CODE4ME_FLOW_ADMIN_PASSWORD || "Code4me-dev1", +}; +const SECRET_REF = process.env.CODE4ME_FLOW_SECRET_REF || "OPENAI_API_KEY"; +const CHANNEL = process.env.CODE4ME_FLOW_CHANNEL ?? "chrome"; +// The plugin repository ships no agent recipe, so the manifest must be named. +const MANIFEST_PATH = process.env.CODE4ME_FLOW_MANIFEST + ? path.resolve(process.env.CODE4ME_FLOW_MANIFEST) + : ""; + +const results = []; +const httpLog = []; + +function record(id, ok, detail = "") { + results.push({ id, status: ok ? "PASS" : "FAIL", detail }); + console.log(`${ok ? "PASS" : "FAIL"} ${id}${detail ? ` :: ${detail}` : ""}`); +} + +function loadPlaywright() { + if (process.env.CODE4ME_PLAYWRIGHT_MODULE) { + return require(process.env.CODE4ME_PLAYWRIGHT_MODULE); + } + try { + return require("playwright"); + } catch (_) { + /* fall through to the npx cache */ + } + const npxRoot = path.join(os.homedir(), ".npm", "_npx"); + if (fs.existsSync(npxRoot)) { + for (const entry of fs.readdirSync(npxRoot)) { + const candidate = path.join(npxRoot, entry, "node_modules", "playwright"); + if (fs.existsSync(candidate)) { + try { + return require(candidate); + } catch (_) { + /* try the next cached install */ + } + } + } + } + throw new Error( + "Playwright not found. Install it or set CODE4ME_PLAYWRIGHT_MODULE to an existing install.", + ); +} + +async function apiRequest(pathname, { method = "GET", body, cookie } = {}) { + const response = await fetch(`${API}${pathname}`, { + method, + headers: { + "Content-Type": "application/json", + ...(cookie ? { Cookie: cookie } : {}), + }, + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), + }); + const text = await response.text(); + let parsed = {}; + try { + parsed = text ? JSON.parse(text) : {}; + } catch (_) { + parsed = { raw: text }; + } + return { status: response.status, body: parsed, headers: response.headers }; +} + +async function loginApi(account) { + const response = await apiRequest("/api/user/authenticate", { + method: "POST", + body: { email: account.email, password: account.password }, + }); + const setCookie = response.headers.get("set-cookie") || ""; + const token = /auth_token=([^;]+)/.exec(setCookie); + return { status: response.status, cookie: token ? `auth_token=${token[1]}` : "" }; +} + +async function createAccount(prefix, configId = 1) { + const stamp = Date.now(); + const account = { + email: `${prefix}-${stamp}@local.dev`, + name: `${prefix} Flow`, + password: "FlowPass123", + config_id: configId, + }; + const response = await apiRequest("/api/user/create", { + method: "POST", + body: account, + }); + return { account, status: response.status, userId: response.body.user_id }; +} + +async function loginUi(page, account) { + await page.goto(`${UI}/login`, { waitUntil: "domcontentloaded" }); + await page.fill("#email", account.email); + await page.fill("#password", account.password); + await Promise.all([ + page.waitForURL((url) => !String(url).includes("/login"), { timeout: 30000 }), + page.click('button[type="submit"]'), + ]); +} + +function trackApi(context) { + context.on("response", (response) => { + const url = response.url(); + if (url.includes("/api/research/") || url.includes("/api/agent/")) { + httpLog.push({ method: response.request().method(), url, status: response.status() }); + } + }); +} + +async function main() { + const { chromium } = loadPlaywright(); + if (!MANIFEST_PATH || !fs.existsSync(MANIFEST_PATH)) { + throw new Error( + "set CODE4ME_FLOW_MANIFEST to a producer manifest (native-.json from " + + "research.study.agents.participant_release native, with its ZIP beside it)", + ); + } + const sourceManifest = JSON.parse(fs.readFileSync(MANIFEST_PATH, "utf8")); + const manifestDir = path.dirname(MANIFEST_PATH); + const manifest = sourceManifest; + const manifestText = JSON.stringify(manifest); + // The producer owns test results: an untested manifest can never be imported, + // so fail with the exact producer command instead of a server-side rejection. + const untested = (manifest.artifacts || []).filter((artifact) => { + const tests = artifact.tests || {}; + return ( + tests.self_check !== "PASS" || + tests.acp_initialize !== "PASS" || + !tests.ran_at + ); + }); + if (!manifest.artifacts || !manifest.artifacts.length || untested.length) { + throw new Error( + `manifest has no passing producer tests: ${MANIFEST_PATH}\n` + + "Produce it first, then point CODE4ME_FLOW_MANIFEST at the result:\n" + + " PYTHONPATH=src python -m research.study.agents.participant_release native \\\n" + + " --skip-build --version --platform macos-arm64 \\\n" + + ' --server-commit "$(git rev-parse HEAD)" --output dist/release-', + ); + } + const archivePaths = manifest.artifacts.map((artifact) => + path.join(manifestDir, artifact.archive), + ); + for (const archivePath of archivePaths) { + if (!fs.existsSync(archivePath)) { + throw new Error(`manifest archive is missing on disk: ${archivePath}`); + } + } + + // ---- setup through the public API ---- + const adminLogin = await loginApi(ADMIN); + record("SETUP-1 admin API login", adminLogin.status === 200, `status=${adminLogin.status}`); + const researcher = await createAccount("flow-researcher"); + record("SETUP-2 create researcher account", researcher.status === 201, `status=${researcher.status}`); + const participant = await createAccount("flow-participant"); + record("SETUP-3 create participant account", participant.status === 201, `status=${participant.status}`); + + const launchOptions = { headless: true }; + if (CHANNEL) launchOptions.channel = CHANNEL; + const browser = await chromium.launch(launchOptions); + const connectionLabel = `flow-conn-${Date.now()}`; + const profileName = `flow-profile-${Date.now()}`; + const studyName = `Flow study ${Date.now()}`; + let releaseId = ""; + let joinCode = ""; + let enrollmentId = ""; + + const adminCtx = await browser.newContext(); + const researcherCtx = await browser.newContext(); + const participantCtx = await browser.newContext(); + trackApi(adminCtx); + trackApi(researcherCtx); + trackApi(participantCtx); + + const admin = await adminCtx.newPage(); + const researcherPage = await researcherCtx.newPage(); + const participantPage = await participantCtx.newPage(); + + try { + // ---- 1. admin -> admin-researchers ---- + await loginUi(admin, ADMIN); + await admin.goto(`${UI}/dashboard?view=admin-researchers`, { waitUntil: "domcontentloaded" }); + await admin.waitForSelector("h2#admin-researchers-title", { timeout: 20000 }); + await admin.waitForSelector(`text=${researcher.account.email}`, { timeout: 20000 }); + const toggle = admin.getByRole("checkbox", { + name: `Researcher access for ${researcher.account.email}`, + }); + // The toggle is controlled and refetches after the PUT, so click (rather + // than check) and wait for the server-backed result. + await toggle.click(); + await admin.waitForSelector("text=now enabled for research", { timeout: 20000 }); + const enableCall = httpLog.find( + (entry) => + entry.method === "PUT" && + entry.url.includes(`/api/research/researchers/${researcher.userId}`), + ); + record( + "ADMIN-1 enable researcher via admin-researchers", + Boolean(enableCall) && enableCall.status === 200, + `PUT status=${enableCall ? enableCall.status : "missing"}`, + ); + + // ---- 2. admin -> admin-connections ---- + await admin.goto(`${UI}/dashboard?view=admin-connections`, { waitUntil: "domcontentloaded" }); + await admin.waitForSelector("h2#admin-connections-title", { timeout: 20000 }); + await admin.getByLabel("Label", { exact: true }).fill(connectionLabel); + await admin.getByLabel("Base URL").fill("http://127.0.0.1:11434/v1"); + await admin.getByLabel("Secret name (env var)").fill(SECRET_REF); + await admin.getByLabel("Models (one per line)").fill("gpt-4o-mini"); + const createConnectionCall = admin.waitForResponse( + (response) => + response.request().method() === "POST" && + response.url().includes("/api/research/provider-connections"), + { timeout: 20000 }, + ); + await admin.getByRole("button", { name: "Create connection" }).click(); + const connectionResponse = await createConnectionCall; + await admin.waitForSelector("text=Provider connection created.", { timeout: 20000 }); + await admin.waitForSelector("text=Ready — secret present in deployment", { timeout: 20000 }); + record( + "ADMIN-2 create ready provider connection", + connectionResponse.status() === 201, + `POST status=${connectionResponse.status()} secret_ref=${SECRET_REF} (name only)`, + ); + + // ---- 3. admin -> admin-agents ---- + await admin.goto(`${UI}/dashboard?view=admin-agents`, { waitUntil: "domcontentloaded" }); + await admin.waitForSelector("h2#admin-agents-title", { timeout: 20000 }); + await admin.getByLabel("Runtime manifest JSON").fill(manifestText); + // Without the declared archive bytes the import button stays disabled: there + // is no digest-trusting acceptance path. + const declaredPlatforms = manifest.artifacts.length; + await admin.waitForSelector("text=Awaiting upload", { timeout: 20000 }); + const awaitingBadges = await admin.getByText("Awaiting upload").count(); + const importDisabled = await admin + .getByRole("button", { name: "Verify and import" }) + .isDisabled(); + record( + "ADMIN-3 declared archives must be uploaded before import", + awaitingBadges === declaredPlatforms && importDisabled, + `awaiting=${awaitingBadges} expected=${declaredPlatforms} disabled=${importDisabled}`, + ); + + await admin.locator('input[aria-label="Agent archive files"]').setInputFiles(archivePaths); + await admin.getByText("Uploaded — not yet verified").first().waitFor({ timeout: 20000 }); + + const importCall = admin.waitForResponse( + (response) => + response.request().method() === "POST" && + response.url().includes("/api/research/agents/releases/import"), + { timeout: 60000 }, + ); + await admin.getByRole("button", { name: "Verify and import" }).click(); + const importResponse = await importCall; + const importBody = importResponse.json ? await importResponse.json() : {}; + releaseId = importBody.release ? importBody.release.release_id : ""; + await admin.waitForSelector("text=Manifest accepted.", { timeout: 20000 }); + const verifiedArtifacts = (importBody.verified_artifacts || []).length; + record( + "ADMIN-4 import verified manifest + archives", + Boolean(releaseId) && + verifiedArtifacts === declaredPlatforms && + (importResponse.status() === 201 || importResponse.status() === 200), + `POST status=${importResponse.status()} release_id=${releaseId} verified=${verifiedArtifacts}`, + ); + + await admin.getByLabel("Platform tests").waitFor({ timeout: 20000 }); + const qualified = await admin + .getByText("QUALIFIED", { exact: true }) + .first() + .waitFor({ timeout: 20000 }) + .then(() => true) + .catch(() => false); + record("ADMIN-6 release qualified from producer tests", qualified, `release_id=${releaseId}`); + + // ---- 4. researcher -> profile (existing page) ---- + await loginUi(researcherPage, researcher.account); + await researcherPage.goto(`${UI}/dashboard?view=agent-profiles`, { waitUntil: "domcontentloaded" }); + await researcherPage.waitForSelector("h2", { timeout: 20000 }); + await researcherPage.getByLabel("Profile name").fill(profileName); + await researcherPage.waitForFunction( + (label) => + Array.from( + document.querySelectorAll('select[name="connection_id"] option'), + ).some((option) => option.textContent.includes(label)), + connectionLabel, + { timeout: 20000 }, + ); + await researcherPage.selectOption('select[name="connection_id"]', { + label: connectionLabel, + }); + await researcherPage.waitForFunction( + (value) => + Boolean( + document.querySelector(`select[name="model"] option[value="${value}"]`), + ), + "gpt-4o-mini", + { timeout: 20000 }, + ); + await researcherPage.selectOption('select[name="model"]', "gpt-4o-mini"); + await researcherPage.waitForFunction( + (value) => + Boolean( + document.querySelector(`select[name="release_id"] option[value="${value}"]`), + ), + releaseId, + { timeout: 20000 }, + ); + await researcherPage.selectOption('select[name="release_id"]', releaseId); + await researcherPage.selectOption('select[name="approval_policy"]', { label: "Ask per step" }); + const profileCall = researcherPage.waitForResponse( + (response) => + response.request().method() === "POST" && + response.url().includes("/api/agent/profiles"), + { timeout: 20000 }, + ); + await researcherPage.getByRole("button", { name: "Create Profile" }).click(); + const profileResponse = await profileCall; + const profileStatus = profileResponse.status(); + if (profileStatus !== 201) { + throw new Error( + `profile create returned ${profileStatus}: ${(await profileResponse.text()).slice(0, 300)}`, + ); + } + // The success notice is cleared by the form reset, so confirm the server + // row instead: the new profile appears in the profiles table. + await researcherPage + .getByText(profileName, { exact: true }) + .waitFor({ timeout: 20000 }); + record( + "RESEARCH-1 create profile pinning release + connection + verified approval", + profileResponse.status() === 201, + `POST status=${profileResponse.status()} profile=${profileName}`, + ); + + // ---- 5. researcher -> create study and read the join code ---- + await researcherPage.goto(`${UI}/research/studies`, { waitUntil: "domcontentloaded" }); + await researcherPage.waitForSelector("h2#research-studies-title", { timeout: 20000 }); + await researcherPage.getByRole("button", { name: "New study" }).click(); + await researcherPage.locator("form.research-card input").first().fill(studyName); + await researcherPage.getByRole("checkbox", { name: profileName }).check(); + const studyCall = researcherPage.waitForResponse( + (response) => + response.request().method() === "POST" && + response.url().endsWith("/api/research/studies"), + { timeout: 20000 }, + ); + await researcherPage.getByRole("button", { name: "Create Draft study" }).click(); + const studyResponse = await studyCall; + await researcherPage.waitForSelector("text=Study created in Draft state.", { timeout: 20000 }); + const studiesList = await researcherCtx.request.get(`${API}/api/research/studies`); + const studiesJson = await studiesList.json(); + const created = (studiesJson.studies || []).find((study) => study.name === studyName); + joinCode = created ? created.join_code : ""; + record( + "RESEARCH-2 create study with fixed profile and read join code", + studyResponse.status() === 201 && Boolean(joinCode), + `POST status=${studyResponse.status()} join_code=${joinCode}`, + ); + + // ---- 6. participant -> web join ---- + await loginUi(participantPage, participant.account); + await participantPage.goto(`${UI}/research/join`, { waitUntil: "domcontentloaded" }); + await participantPage.locator("input[required]").fill(joinCode); + await participantPage.getByRole("button", { name: "Review study" }).click(); + await participantPage.waitForSelector("text=I accept the study consent notice.", { + timeout: 20000, + }); + const joinCall = participantPage.waitForResponse( + (response) => + response.request().method() === "POST" && + response.url().endsWith("/api/research/join"), + { timeout: 20000 }, + ); + await participantPage.locator('input[type="checkbox"]').first().check(); + await participantPage.getByRole("button", { name: "Accept and join" }).click(); + const joinResponse = await joinCall; + await participantPage.waitForSelector('div[aria-label="Enrollment handoff"]', { + timeout: 20000, + }); + enrollmentId = await participantPage + .locator('div[aria-label="Enrollment handoff"]') + .getAttribute("data-enrollment-id"); + record( + "PARTICIPANT-1 web join with consent", + joinResponse.status() === 201 && Boolean(enrollmentId), + `POST status=${joinResponse.status()} enrollment=${enrollmentId}`, + ); + + // ---- 7. bootstrap before vs after consent (HTTP) ---- + const preConsent = await participantCtx.request.post( + `${API}/api/research/bootstrap/research-sessions`, + { + data: { + enrollment_id: randomUUID(), + context_id: "flow-pre-consent", + environment: { os: "macos", arch: "arm64" }, + }, + }, + ); + const afterConsent = await participantCtx.request.post( + `${API}/api/research/bootstrap/research-sessions`, + { + data: { + enrollment_id: enrollmentId, + context_id: "flow-post-consent", + environment: { os: "macos", arch: "arm64" }, + }, + }, + ); + record( + "PARTICIPANT-2 cannot bootstrap before consent", + preConsent.status() === 404, + `POST status=${preConsent.status()} (no enrollment exists yet)`, + ); + record( + "PARTICIPANT-3 can bootstrap after consent", + afterConsent.status() === 201, + `POST status=${afterConsent.status()}`, + ); + } catch (error) { + record("HARNESS", false, String(error && error.message ? error.message : error).slice(0, 400)); + } finally { + await browser.close(); + } + + const failed = results.filter((entry) => entry.status !== "PASS"); + console.log(`\nTOTAL ${results.length} PASS ${results.length - failed.length} FAIL ${failed.length}`); + failed.forEach((entry) => console.log(` FAILED ${entry.id} ${entry.detail}`)); + const report = { steps: results, http: httpLog }; + if (process.env.CODE4ME_FLOW_REPORT) { + fs.writeFileSync(process.env.CODE4ME_FLOW_REPORT, JSON.stringify(report, null, 2)); + } + return failed.length ? 1 : 0; +} + +main() + .then((code) => process.exit(code)) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/e2e/browser/scenarios.py b/e2e/browser/scenarios.py new file mode 100644 index 0000000..3aba777 --- /dev/null +++ b/e2e/browser/scenarios.py @@ -0,0 +1,658 @@ +"""Task 07 §6 browser scenarios A–E against the disposable e2e stack. + +Uses the installed Playwright Chromium via the Python API. Each run creates a +fresh participant account through the public API so the one-active-enrollment +rule never confounds the scenario, and targets studies by captured id (not by +name matching). Adds no repository dependency; never touches the dev stack. + +Every check has one stable id from the runner's exact inventory; the runner +refuses to pass when an id is missing, duplicated, blocked or unexpected. Join +codes and account credentials are never printed or written to the results. + +Beyond the visible UI, the checks re-read the real API models: the edited study +metadata, the stopped study's retained enrollment/assignment counts, the +cloned study's identity/profile selection/status, the participant's own +enrollment projection and the owner-scoped participant coverage model. + +A: researcher create/edit/lock, B: participant web join, C: stop/clone, +D: authorization, E: study workspace tabs (participants, dashboard, analytics), +participant budgets (spent/budget columns, a top-up from the drawer re-read +through the budget API) and the participant's My studies page. +""" + +from __future__ import annotations + +import json +import os +import re +import sys +import time +import urllib.request +from pathlib import Path + +from playwright.sync_api import sync_playwright + +config_path = os.environ.get("CODE4ME_E2E_BROWSER_CONFIG") +CONFIG = json.loads(Path(config_path).read_text()) if config_path else {} +UI = CONFIG.get("ui_url", "http://localhost:3900") +API = CONFIG.get("base_url", "http://localhost:28008") +OWNER = CONFIG.get("researcher", {"email": "e2e-researcher@example.com", "password": "ResearcherPass123"}) +ADMIN = CONFIG.get("admin", {"email": "e2e-admin@example.com", "password": "AdminPass123"}) +PROFILE_NAME = CONFIG.get("profile_name", "e2e-profile") +EXPECTED_IDS = tuple( + str(item) for item in json.loads(os.environ.get("CODE4ME_E2E_BROWSER_EXPECTED_IDS", "[]")) +) + +results: list[dict[str, str]] = [] + + +def record(step_id: str, title: str, ok: bool, detail: str = "") -> None: + status = "PASS" if ok else "FAIL" + results.append({"id": step_id, "title": title, "status": status, "detail": detail}) + print(f"{status} {step_id} {title}" + (f" :: {detail}" if detail else "")) + + +def record_blocked(step_id: str, title: str, reason: str) -> None: + results.append({"id": step_id, "title": title, "status": "BLOCKED", "detail": reason}) + print(f"BLOCKED {step_id} {title} :: {reason}") + + +def api_post(path: str, payload: dict, token: str | None = None) -> dict: + request = urllib.request.Request( + f"{API}{path}", + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", **({"Authorization": f"Bearer {token}"} if token else {})}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=20) as response: + return json.loads(response.read() or b"{}") + + +def fresh_participant() -> dict: + stamp = int(time.time() * 1000) + account = { + "email": f"e2e-browser-{stamp}@example.com", + "password": "BrowserPass123", + "name": "Browser Participant", + } + api_post("/api/user/create", {**account, "config_id": CONFIG.get("config_id", 1)}) + return account + + +def login(page, account) -> None: + page.goto(f"{UI}/login", wait_until="networkidle") + page.fill("#email", account["email"]) + page.fill("#password", account["password"]) + page.click('button[type="submit"]') + page.wait_for_url(lambda url: "/login" not in str(url), timeout=20000) + + +def find_study_id(page, name: str) -> str | None: + return page.evaluate( + """async (name) => { + const res = await fetch('/api/research/studies', { credentials: 'include' }); + const data = await res.json(); + return (data.studies || []).find((s) => s.name === name)?.study_id || null; + }""", + name, + ) + + +def study_detail(page, study_id: str) -> dict: + return page.evaluate( + """async (id) => { + const res = await fetch(`/api/research/studies/${id}`, { credentials: 'include' }); + if (!res.ok) return {}; + const data = await res.json(); + return data.study || {}; + }""", + study_id, + ) + + +def participant_budget(page, study_id: str, enrollment_id: str | None) -> dict: + """The owner-scoped budget of one enrollment (resolved from the table when unknown).""" + return page.evaluate( + """async ({ id, enrollmentId }) => { + let eid = enrollmentId; + if (!eid) { + const rows = await fetch(`/api/research/studies/${id}/analytics/participants`, { credentials: 'include' }); + let data = {}; + try { data = await rows.json(); } catch (_) { data = {}; } + eid = ((data.participants || [])[0] || {}).enrollment_id || null; + } + if (!eid) return {}; + const res = await fetch(`/api/research/studies/${id}/enrollments/${eid}/budget`, { credentials: 'include' }); + if (!res.ok) return {}; + return await res.json(); + }""", + {"id": study_id, "enrollmentId": enrollment_id}, + ) + + +def participant_coverage(page, study_id: str) -> dict: + return page.evaluate( + """async (id) => { + const res = await fetch( + `/api/research/operations/participants/coverage?study_id=${encodeURIComponent(id)}`, + { credentials: 'include' }); + let data = {}; + try { data = await res.json(); } catch (_) { data = {}; } + return { status: res.status, data }; + }""", + study_id, + ) + + +def open_tab(details, name: str) -> None: + """Select a study workspace tab; tab panels only exist while selected.""" + details.get_by_role("tab", name=re.compile(name, re.I)).click() + + +def main() -> int: + participant_account = fresh_participant() + study_id: str | None = None + clone_id: str | None = None + enrollment_id: str | None = None + assignment_id: str | None = None + profile_id: str | None = None + renamed = "" + edited_description = "" + clone_name = "" + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=True) + owner_ctx = browser.new_context() + participant_ctx = browser.new_context() + owner = owner_ctx.new_page() + participant = participant_ctx.new_page() + + captured: list[dict] = [] + owner.on( + "request", + lambda req: captured.append({"method": req.method, "url": req.url, "body": req.post_data}) + if "/api/research/" in req.url + else None, + ) + + try: + # ---- Scenario A: researcher create + metadata lock ---- + login(owner, OWNER) + owner.goto(f"{UI}/research/studies", wait_until="networkidle") + record("A1", "owner reaches the research control plane", owner.locator("h2#research-studies-title").is_visible()) + record( + "A2", + "no revision/draft/supersede authoring controls render", + owner.get_by_text(re.compile(r"revision|supersede|publish draft", re.I)).count() == 0, + ) + + owner.click('button:has-text("New study")') + study_name = f"Browser study {int(time.time())}" + owner.locator("form.research-card input").first.fill(study_name) + owner.locator("fieldset.research-profile-selection input[type=checkbox]").first.check() + # The seeded arm runs the built-in agent (metered), so the form + # requires a default budget per participant before it submits. + owner.locator("#study-default-budget").wait_for(timeout=20000) + owner.locator("#study-default-budget").fill("5") + owner.click('button:has-text("Create Draft study")') + owner.wait_for_selector("text=Study created in Draft state.", timeout=20000) + + create_req = next( + (r for r in captured if r["method"] == "POST" and r["url"].endswith("/api/research/studies")), + None, + ) + create_body = json.loads(create_req["body"]) if create_req and create_req["body"] else {} + record( + "A3", + "create posts the complete setup once (with profiles and the default budget)", + bool(create_req) + and len(create_body.get("profile_ids", [])) >= 1 + # The form normalises the amount ("5" → "5.00"); compare the value, not the spelling. + and str(create_body.get("default_budget_usd") or "").replace(",", "") in ("5", "5.0", "5.00"), + ) + record("A4", "create body carries no revision/condition key", bool(create_req) and not re.search(r"revision|condition", create_req["body"])) + + study_id = find_study_id(owner, study_name) + owner.locator("button.research-list-item", has_text=study_name).click() + details = owner.locator('section[aria-label="Study details"]') + details.wait_for() + join_code = details.locator("dd").nth(1).inner_text().strip() + record("A5", "DRAFT shows a join code and the fixed profile summary", bool(re.match(r"^[A-F0-9]{6,}$", join_code))) + record("A6", "profile selection is presented as fixed", details.get_by_text("Profile selection is fixed after study creation.").is_visible()) + record("A7", "study details contain no revision ID", not re.search(r"revision", details.inner_text(), re.I)) + + renamed = f"{study_name} (edited)" + edited_description = f"Edited description for {study_name}" + # Metadata editing lives in the study's Settings tab. + open_tab(details, "settings") + details.locator("form input").first.fill(renamed) + details.locator("form textarea").first.fill(edited_description) + owner.click('button:has-text("Save metadata")') + owner.wait_for_selector("text=Study metadata updated.", timeout=20000) + record("A8", "metadata edit is allowed before first consent", True) + + persisted_study = study_detail(owner, study_id) + record( + "A12", + "edited metadata is persisted in the study read model", + persisted_study.get("name") == renamed + and persisted_study.get("description") == edited_description + and not persisted_study.get("consent_locked_at") + and persisted_study.get("research_status") == "DRAFT", + ) + + # ---- Scenario B: participant web-only join ---- + participant.goto(f"{UI}/research/join", wait_until="networkidle") + participant.locator("input[required]").fill(join_code) + participant.click('button:has-text("Review study")') + participant.wait_for_url(re.compile(r"/login$"), timeout=20000) + record("B1", "unauthenticated join redirects to login preserving intent", True) + + login(participant, participant_account) + participant.wait_for_url(re.compile(r"/research/join"), timeout=20000) + participant.wait_for_function("() => document.querySelector('input[required]').value.length > 0", timeout=20000) + prefilled = participant.locator("input[required]").input_value() + record("B2", "login returns to the join page with the code preserved", prefilled == join_code) + participant.click('button:has-text("Review study")') + participant.wait_for_selector("text=I accept the study consent notice.", timeout=20000) + record("B3", "safe study summary and consent control render", True) + record("B4", "participant sees exactly one checkbox (consent, no profile picker)", participant.locator('input[type="checkbox"]').count() == 1) + + participant.click('button:has-text("Accept and join")') + participant.wait_for_selector("text=You must accept the consent notice before joining.", timeout=20000) + record("B5", "consent-unchecked join is refused", True) + + participant.locator('input[type="checkbox"]').first.check() + participant.click('button:has-text("Accept and join")') + participant.wait_for_selector('div[aria-label="Enrollment handoff"]', timeout=20000) + handoff = participant.locator('div[aria-label="Enrollment handoff"]') + enrollment_id = handoff.get_attribute("data-enrollment-id") + assignment_id = handoff.get_attribute("data-assignment-id") + profile_id = handoff.get_attribute("data-agent-profile-id") + record( + "B6", + "exactly one enrollment and assignment handed off", + bool(enrollment_id and assignment_id and profile_id), + "enrollment, assignment and agent profile ids present" + if enrollment_id and assignment_id and profile_id + else "handoff data attributes incomplete", + ) + record("B7", "success state carries the IntelliJ next step", "post-enrollment activation" in handoff.inner_text()) + + own_status = participant.evaluate( + """async () => { + const res = await fetch('/api/research/participants/me', { credentials: 'include' }); + let data = {}; + try { data = await res.json(); } catch (_) { data = {}; } + return { status: res.status, data }; + }""" + ) + own_rows = [ + row + for row in (own_status.get("data") or {}).get("enrollments", []) + if row.get("study_id") == study_id + ] + record( + "B8", + "participant read model reports the persisted ACTIVE enrollment", + own_status.get("status") == 200 + and len(own_rows) == 1 + and own_rows[0].get("enrollment_id") == enrollment_id + and own_rows[0].get("status") == "ACTIVE", + ) + + # ---- back in A: ACTIVE, metadata locked, stale client rejected ---- + owner.click('button:has-text("Refresh")') + owner.wait_for_function( + """(name) => Array.from(document.querySelectorAll('button.research-list-item')) + .some((b) => b.textContent.includes(name) && b.textContent.includes('Active'))""", + arg=renamed, + timeout=20000, + ) + owner.locator("button.research-list-item", has_text=renamed).click() + # Reselecting a study may reset the workspace to its Overview tab. + open_tab(details, "settings") + owner.wait_for_function( + """() => { + const panel = document.querySelector('section[aria-label="Study details"]'); + if (!panel) return false; + const input = panel.querySelector('form input'); + return Boolean(input) && input.disabled; + }""", + timeout=20000, + ) + study_state = study_detail(owner, study_id) + record( + "A9", + "owner sees ACTIVE with one active enrollment and assignment", + study_state.get("research_status") == "ACTIVE" + and study_state.get("active_enrollment_count") == 1 + and study_state.get("assignment_count") == 1 + and study_state.get("active_assignment_count") == 1, + f"enrollments={study_state.get('enrollment_count')} " + f"active={study_state.get('active_enrollment_count')} " + f"assignments={study_state.get('assignment_count')}", + ) + record("A10", "metadata inputs are disabled after first consent", details.locator("form input").first.is_disabled()) + + forced_status = owner.evaluate( + """async (id) => { + const res = await fetch(`/api/research/studies/${id}/metadata`, { + method: "PATCH", headers: { "Content-Type": "application/json" }, + credentials: "include", body: JSON.stringify({ name: "stale client write" }), + }); + return res.status; + }""", + study_id, + ) + record("A11", "stale-client metadata write is rejected after consent", forced_status == 409, f"status={forced_status}") + + identity_ready = bool(enrollment_id and assignment_id and profile_id) + if identity_ready: + coverage_payload = participant_coverage(owner, study_id) + coverage_data = coverage_payload.get("data") or {} + coverage_rows = [ + row + for row in coverage_data.get("participants", []) + if row.get("enrollment_id") == enrollment_id + ] + coverage_assignment = (coverage_rows[0].get("assignment") if coverage_rows else None) or {} + record( + "A13", + "coverage read model exposes the enrollment and frozen assignment", + coverage_payload.get("status") == 200 + and coverage_data.get("participant_count") == 1 + and len(coverage_rows) == 1 + and coverage_rows[0].get("status") == "ACTIVE" + and coverage_assignment.get("assignment_id") == assignment_id + and coverage_assignment.get("agent_profile_id") == profile_id + and coverage_assignment.get("status") == "ACTIVE" + and bool(coverage_assignment.get("strategy")), + ) + else: + record_blocked("A13", "coverage read model exposes the enrollment and frozen assignment", "handoff identity missing (see B6)") + + # ---- Scenario E: study workspace — participants, dashboards, analytics ---- + open_tab(details, "participants") + participants_table = details.get_by_role("table", name="Enrolled participants") + participants_table.wait_for(timeout=20000) + rows = participants_table.locator("tbody tr") + arm_cell = rows.first.locator("td").nth(1).inner_text().strip() if rows.count() else "" + table_text = participants_table.inner_text() + record( + "E1", + "participants tab lists the enrolled participant, its frozen arm and no account identity", + rows.count() == 1 + and PROFILE_NAME in arm_cell + and "Not assigned" not in arm_cell + and "Active" in table_text + and "@" not in table_text, + f"arm={arm_cell.splitlines()[0] if arm_cell else ''}", + ) + table_lower = table_text.lower() # header cells render uppercased by CSS + record( + "E5", + "participants tab shows the participant's spent amount and budget", + "spent" in table_lower and "budget" in table_lower and "$5.00" in table_text and "$0.00" in table_text, + "table=" + " ".join(table_text.split())[:260], + ) + rows.first.get_by_role("button", name=re.compile("open dashboard", re.I)).click() + drawer = owner.get_by_role("dialog") + drawer.wait_for(timeout=20000) + owner.wait_for_function( + """() => { const d = document.querySelector('[role=dialog]'); + return d && !d.textContent.includes('Loading participant telemetry'); }""", + timeout=20000, + ) + drawer_text = drawer.inner_text() + record( + "E2", + "the participant dashboard opens with metadata-only telemetry", + "Participant" in drawer_text and ("Prompts" in drawer_text or "No telemetry" in drawer_text), + ) + # ---- Budgets: top up from the drawer, then re-read the budget API ---- + drawer.get_by_role("button", name=re.compile("adjust budget", re.I)).click() + adjust_form = drawer.get_by_role("form", name="Adjust budget") + adjust_form.wait_for(timeout=20000) + adjust_form.locator("#adjust-budget-amount").fill("2.50") + adjust_form.locator("#adjust-budget-reason").fill("browser top-up") + adjust_form.get_by_role("button", name=re.compile("top up budget", re.I)).click() + owner.wait_for_selector("text=Topped up by $2.50", timeout=20000) + budget_state = participant_budget(owner, study_id, enrollment_id) + budget_balance = budget_state.get("balance") or {} + record( + "E6", + "owner tops up the participant's budget from the drawer and the ledger records the reason", + budget_balance.get("limit_micro_usd") == 7_500_000 + and budget_balance.get("consumed_micro_usd") == 0 + and any( + item.get("kind") == "TOP_UP" and item.get("reason") == "browser top-up" + for item in budget_state.get("recent_adjustments", []) + ), + f"limit={budget_balance.get('limit_micro_usd')}", + ) + owner.keyboard.press("Escape") + drawer.wait_for(state="detached", timeout=20000) + open_tab(details, "analytics") + owner.wait_for_selector("text=Arm comparison", timeout=20000) + record( + "E3", + "analytics compares arms on participant-level metrics", + owner.get_by_text("Participants with telemetry").count() >= 1, + ) + participant.goto(f"{UI}/research/my-studies", wait_until="networkidle") + participant.wait_for_selector(f"text={renamed}", timeout=20000) + record( + "E4", + "My studies shows the joined study and its schedule", + participant.get_by_text("Study ends").count() >= 1 and participant.locator('input[type="checkbox"]').count() == 0, + ) + record( + "E7", + "My studies shows the remaining budget without any arm detail", + participant.get_by_text("Budget remaining").count() >= 1 + and participant.get_by_text(re.compile(r"provided by the study")).count() >= 1 + and participant.get_by_text("$7.50").count() >= 1 + and participant.get_by_text(re.compile(r"own account|profile", re.I)).count() == 0, + ) + + # ---- Scenario C: stop, retained data, clone ---- + open_tab(details, "settings") + owner.wait_for_selector('button:has-text("Stop study")', state="visible", timeout=20000) + owner.once("dialog", lambda dialog: dialog.accept()) + owner.click('button:has-text("Stop study")') + owner.wait_for_selector("text=Study stopped. Collection is revoked and retained data remains available.", timeout=20000) + record("C1", "stop is terminal and non-destructive in the UI", True) + stopped_state = study_detail(owner, study_id) + record( + "C2", + "stopped study retains its enrollment rows", + stopped_state.get("research_status") == "STUDY_STOPPED" + and stopped_state.get("enrollment_count") == 1, + f"enrollments={stopped_state.get('enrollment_count')}", + ) + + clone_name = f"{renamed} (copy)" + owner.click('button:has-text("Duplicate")') + # The duplicate form renders above the study list, not inside the + # details panel: every field is prefilled and stays editable. + clone_form = owner.locator("form.study-create-form").filter(has_text="Duplicate study") + clone_form.wait_for() + clone_profile = clone_form.get_by_label(PROFILE_NAME, exact=True) + clone_copies = clone_form.get_by_label("Name", exact=True).input_value() == clone_name + record( + "C3", + "duplicate form is prefilled from the stopped study, including its still-active profile", + clone_copies + and "not copied" in clone_form.inner_text() + and clone_profile.count() == 1 + and clone_profile.is_checked(), + ) + if clone_profile.count() != 1: + record_blocked("C4", "clone persists a distinct DRAFT study with the reselected profile", "the agent profile is not offered in the duplicate form") + else: + clone_profile.check() + clone_form.get_by_role("button", name="Create Draft study").click() + owner.wait_for_selector("text=/Duplicate of .* created in Draft state/", timeout=20000) + clone_id = find_study_id(owner, clone_name) + clone_study = study_detail(owner, clone_id) if clone_id else {} + clone_profiles = [item.get("profile_id") for item in clone_study.get("profile_selections") or []] + record( + "C4", + "clone persists a distinct DRAFT study with the reselected profile", + bool(clone_id) + and clone_id != study_id + and clone_study.get("name") == clone_name + and clone_study.get("description") == edited_description + and clone_study.get("research_status") == "DRAFT" + and profile_id in clone_profiles + and clone_study.get("enrollment_count") == 0 + and clone_study.get("assignment_count") == 0 + and bool((clone_study.get("lifecycle_capabilities") or {}).get("joinable")), + ) + + participant.goto(f"{UI}/research/join", wait_until="networkidle") + participant.locator("input[required]").fill(join_code) + participant.click('button:has-text("Review study")') + participant.wait_for_selector("text=/stopped and cannot accept new participants|invalid or no longer available/i", timeout=20000) + record("C5", "stopped study cannot be joined", True) + + if assignment_id: + stopped_coverage = participant_coverage(owner, study_id) + stopped_data = stopped_coverage.get("data") or {} + stopped_rows = [ + row + for row in stopped_data.get("participants", []) + if row.get("enrollment_id") == enrollment_id + ] + stopped_assignment = (stopped_rows[0].get("assignment") if stopped_rows else None) or {} + record( + "C6", + "coverage read model retains the stopped enrollment and assignment", + stopped_coverage.get("status") == 200 + and stopped_data.get("participant_count") == 1 + and len(stopped_rows) == 1 + and stopped_rows[0].get("status") == "STUDY_STOPPED" + and stopped_assignment.get("assignment_id") == assignment_id + and stopped_assignment.get("status") == "STUDY_STOPPED", + ) + else: + record_blocked("C6", "coverage read model retains the stopped enrollment and assignment", "handoff identity missing (see B6)") + + # ---- Scenario D: authorization ---- + participant.goto(f"{UI}/research/studies", wait_until="networkidle") + # Non-researchers are redirected to My studies; older builds showed + # a permission notice instead. Either way no control plane renders. + try: + participant.wait_for_url(re.compile(r"/research/my-studies"), timeout=20000) + redirected = True + except Exception: # noqa: BLE001 - fall back to the notice check + redirected = False + record( + "D1", + "participant cannot reach the researcher control plane", + participant.locator("h2#research-studies-title").count() == 0 + and (redirected or participant.get_by_text("You do not have permission to view research studies.").is_visible()), + "redirected to My studies" if redirected else "", + ) + revoke_status = participant.evaluate( + """async (args) => { + const res = await fetch( + `/api/research/studies/${args.studyId}/enrollments/${args.enrollmentId}/revoke`, + { method: "POST", headers: { "Content-Type": "application/json" }, + credentials: "include", body: "{}" }); + return res.status; + }""", + {"studyId": study_id, "enrollmentId": enrollment_id}, + ) + record("D2", "participant cannot revoke an enrollment", revoke_status in (403, 404), f"status={revoke_status}") + + admin_ctx = browser.new_context() + admin = admin_ctx.new_page() + login(admin, ADMIN) + admin.goto(f"{UI}/research/studies", wait_until="networkidle") + record("D3", "admin reaches the control plane", admin.locator("h2#research-studies-title").is_visible()) + # The kill-switch control lives in the Settings tab of a non-stopped + # study; the clone is the only non-stopped study at this point. + admin_target = clone_name if clone_id else CONFIG.get("study_name", "E2E Synthetic Study") + admin.locator("button.research-list-item", has_text=admin_target).first.click() + admin_details = admin.locator('section[aria-label="Study details"]') + admin_details.wait_for(timeout=20000) + open_tab(admin_details, "settings") + kill_switch = admin.get_by_role("button", name=re.compile("kill switch", re.I)) + try: + # The admin flag loads asynchronously; absence after a wait is + # a failed check, not a harness crash. + kill_switch.first.wait_for(timeout=10000) + kill_switch_present = True + except Exception: # noqa: BLE001 - report absence as a failed check + kill_switch_present = False + record( + "D4", + "admin sees the kill-switch control in the study detail panel", + kill_switch_present, + ) + # Compare like with like: the owner looks at the same Settings tab. + open_tab(details, "settings") + record( + "D5", + "non-admin researcher sees no kill-switch control", + owner.get_by_role("button", name=re.compile("kill switch", re.I)).count() == 0, + ) + coverage_forbidden = participant.evaluate( + """async (id) => { + const res = await fetch( + `/api/research/operations/participants/coverage?study_id=${encodeURIComponent(id)}`, + { credentials: "include" }); + return res.status; + }""", + study_id, + ) + record( + "D6", + "participant cannot read study participant coverage", + coverage_forbidden in (403, 404), + f"status={coverage_forbidden}", + ) + admin_ctx.close() + except Exception as error: # noqa: BLE001 - report and continue + record("harness", "unhandled browser harness error", False, str(error).replace("\n", " ")[:300]) + finally: + browser.close() + + context = { + "study_id": study_id or "", + "edited_name": renamed, + "edited_description": edited_description, + "clone_study_id": clone_id or "", + "clone_name": clone_name, + "profile_id": profile_id or "", + "enrollment_id": enrollment_id or "", + "assignment_id": assignment_id or "", + } + ids = [item["id"] for item in results] + duplicates = sorted({step_id for step_id in ids if ids.count(step_id) > 1}) + missing = [step_id for step_id in EXPECTED_IDS if step_id not in ids] + unexpected = [step_id for step_id in ids if step_id not in EXPECTED_IDS] + non_pass = [item for item in results if item["status"] != "PASS"] + if EXPECTED_IDS: + inventory_ok = not missing and not unexpected and not duplicates and len(results) == len(EXPECTED_IDS) + else: + inventory_ok = not duplicates + print(f"\nTOTAL {len(results)} PASS {len(results) - len(non_pass)} NON-PASS {len(non_pass)}") + for item in non_pass: + print(f" {item['status']} {item['id']} {item['title']} {item['detail']}") + if not inventory_ok: + print(f" INVENTORY missing={missing} unexpected={unexpected} duplicated={duplicates}") + result_path = os.environ.get("CODE4ME_E2E_BROWSER_RESULTS") + if result_path: + Path(result_path).write_text(json.dumps({ + "steps": results, + "context": context, + "inventory": {"expected": list(EXPECTED_IDS), "missing": missing, + "unexpected": unexpected, "duplicated": duplicates}, + }, indent=2)) + return 0 if (not non_pass and inventory_ok) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/e2e/browser/serve.js b/e2e/browser/serve.js new file mode 100644 index 0000000..c15dc9d --- /dev/null +++ b/e2e/browser/serve.js @@ -0,0 +1,71 @@ +// Isolated browser-test host for the Code4Me research UI. +// Serves the built SPA from code4me2-server/src/website/build and proxies /api +// to the disposable e2e backend (localhost:28008), so the browser is strictly +// same-origin and no dev stack or repo file is touched. +const http = require("http"); +const fs = require("fs"); +const path = require("path"); + +const BUILD = path.resolve(process.env.CODE4ME_E2E_WEB_BUILD || path.join(__dirname, "../code4me2-server/src/website/build")); +const BACKEND = new URL(process.env.CODE4ME_E2E_BASE_URL || "http://127.0.0.1:28008"); +const PORT = Number(process.env.CODE4ME_E2E_WEB_PORT || 3900); + +const TYPES = { + ".html": "text/html; charset=utf-8", + ".js": "application/javascript; charset=utf-8", + ".css": "text/css; charset=utf-8", + ".json": "application/json; charset=utf-8", + ".svg": "image/svg+xml", + ".png": "image/png", + ".ico": "image/x-icon", + ".map": "application/json", +}; + +function serveFile(res, filePath) { + fs.readFile(filePath, (err, data) => { + if (err) { + res.writeHead(404, { "Content-Type": "text/plain" }); + res.end("not found"); + return; + } + res.writeHead(200, { "Content-Type": TYPES[path.extname(filePath)] || "application/octet-stream" }); + res.end(data); + }); +} + +const server = http.createServer((req, res) => { + const url = new URL(req.url, `http://localhost:${PORT}`); + if (url.pathname.startsWith("/api/") || url.pathname === "/api") { + const proxied = http.request( + { + host: BACKEND.hostname, + port: BACKEND.port, + method: req.method, + path: req.url, + headers: { ...req.headers, host: BACKEND.host }, + }, + (upstream) => { + res.writeHead(upstream.statusCode || 502, upstream.headers); + upstream.pipe(res); + } + ); + proxied.on("error", () => { + res.writeHead(502, { "Content-Type": "application/json" }); + res.end('{"detail":"proxy upstream unavailable"}'); + }); + req.pipe(proxied); + return; + } + + const candidate = path.join(BUILD, url.pathname); + if (url.pathname !== "/" && fs.existsSync(candidate) && fs.statSync(candidate).isFile()) { + serveFile(res, candidate); + return; + } + // SPA fallback for client-side routes such as /research/join. + serveFile(res, path.join(BUILD, "index.html")); +}); + +server.listen(PORT, "127.0.0.1", () => { + console.log(`research-ui-test-host listening on http://127.0.0.1:${PORT} -> backend ${BACKEND.origin}`); +}); diff --git a/e2e/code4me_e2e/__init__.py b/e2e/code4me_e2e/__init__.py new file mode 100644 index 0000000..3e34e5e --- /dev/null +++ b/e2e/code4me_e2e/__init__.py @@ -0,0 +1,9 @@ +"""Standalone, parameterized end-to-end workflow harness for Code4Me. + +Nothing in this package imports the plugin or the server: it drives the real +HTTP API of a disposable backend stack it provisions itself. +""" + +__all__ = ["__version__"] + +__version__ = "0.1.0" diff --git a/e2e/code4me_e2e/__main__.py b/e2e/code4me_e2e/__main__.py new file mode 100644 index 0000000..63667ff --- /dev/null +++ b/e2e/code4me_e2e/__main__.py @@ -0,0 +1,6 @@ +"""``python3 -m code4me_e2e`` entry point.""" + +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/e2e/code4me_e2e/acp.py b/e2e/code4me_e2e/acp.py new file mode 100644 index 0000000..60a2a78 --- /dev/null +++ b/e2e/code4me_e2e/acp.py @@ -0,0 +1,371 @@ +"""A minimal ACP host that launches the real IDE-registered proxy over stdio. + +No credentials are synthesized here: the registered agent uses the plugin's +managed authentication bridge. Only the paid model provider is deterministic. + +Every assertion is scoped to *this* invocation: the registered entry's exact +command/argv/digest/version, the monotonic stub receipt delta for the first +prompt, and telemetry rows emitted no earlier than this launch by this emitter +(or by the IDE spool for this run). +""" +from __future__ import annotations + +import json +import os +import queue +import subprocess +import threading +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from . import process, real_agents, stack + +#: ACP method label -> canonical event type the proxy normalizer persists +#: (``research/telemetry/normalization/generic_acp.py``). The ACP method label +#: itself is a normalizer rule id and is not stored in ``research_event``, so +#: the proof is the canonical projection, not the label. +ACP_METHOD_EVENT_TYPES: Dict[str, str] = { + "initialize": "interaction.started", + "session/new": "interaction.started", + "session/prompt": "agent.message.started", +} + +#: Minimum persisted canonical counts for the ACP proxy emitter of this launch: +#: ``initialize`` + ``session/new`` -> two interaction.started; the prompt +#: request plus at least one assistant chunk -> two agent.message.started; the +#: first prompt response -> one agent.message.completed. +REQUIRED_ACP_EVENT_COUNTS: Dict[str, int] = { + "interaction.started": 2, + "agent.message.started": 2, + "agent.message.completed": 1, +} + +#: Sources that must both be present, scoped to this invocation's window. +REQUIRED_TELEMETRY_SOURCES = ("ide", "acp") + + +class AcpClient: + def __init__(self, child: subprocess.Popen): + self.child = child + self.messages = queue.Queue() + self.chunks: list[str] = [] + self.methods: List[str] = [] + self.sequence = 0 + threading.Thread(target=self._read, daemon=True).start() + + def _read(self): + try: + for line in self.child.stdout: + self.messages.put(json.loads(line)) + except Exception as error: + self.messages.put(error) + finally: + self.messages.put(EOFError("ACP process closed stdout")) + + def send(self, payload: dict): + self.child.stdin.write(json.dumps({"jsonrpc": "2.0", **payload}) + "\n") + self.child.stdin.flush() + + def request(self, method: str, params: dict, timeout: int = 120) -> dict: + self.sequence += 1 + request_id = self.sequence + self.methods.append(method) + self.send({"id": request_id, "method": method, "params": params}) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + message = self.messages.get(timeout=max(0.01, deadline - time.monotonic())) + except queue.Empty: + break + if isinstance(message, Exception): + raise message + if message.get("method") == "session/update": + update = message.get("params", {}).get("update", {}) + if update.get("sessionUpdate") == "agent_message_chunk": + self.chunks.append(update.get("content", {}).get("text", "")) + elif "method" in message and "id" in message: + # This deterministic turn needs no host tools. Unexpected host + # calls fail closed instead of executing commands on the machine. + self.send({"id": message["id"], "error": {"code": -32601, "message": "Unsupported host method"}}) + elif message.get("id") == request_id: + if "error" in message: + raise RuntimeError(f"ACP {method} failed: {message['error']}") + return message.get("result", {}) + raise TimeoutError(f"ACP {method} did not finish within {timeout}s") + + +# --------------------------------------------------------------------------- +# Registered entry identity +# --------------------------------------------------------------------------- + + +def registered_entry(run_path: Path) -> Tuple[Path, dict]: + """The single research-proxy ACP entry this run registered.""" + registry_path = run_path / "ide-home/.jetbrains/acp.json" + if not registry_path.is_file(): + raise RuntimeError(f"registered ACP registry {registry_path} does not exist") + try: + registry = json.loads(registry_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"registered ACP registry {registry_path} is unreadable: {error}") from error + entries = [entry for name, entry in (registry.get("agent_servers") or {}).items() + if name.startswith("Code4Me Research Proxy")] + if len(entries) != 1: + raise RuntimeError(f"Expected one research proxy registration, found {len(entries)}") + return registry_path, entries[0] + + +def _flag_value(args: Sequence[str], flag: str) -> Optional[str]: + """The value following ``flag`` in a proxy argv, if any.""" + for index, token in enumerate(args[:-1]): + if token == flag: + return args[index + 1] + return None + + +def _normalize_digest(value: Optional[str]) -> Optional[str]: + """Bare lowercase hex; ``sha256:`` is a display convention, not identity.""" + if not value: + return None + text = str(value).strip().lower() + if text.startswith("sha256:"): + text = text[len("sha256:"):] + return text or None + + +def entry_identity(run_path: Path) -> Dict[str, Any]: + """Identify and validate the registered entry and the agent it wraps. + + Records command/argv/env keys plus the executable digest and ``--version`` + of both the proxy and the real agent. Environment *values* are never + recorded: the entry env carries the one-time spool capability. + """ + registry_path, entry = registered_entry(run_path) + command = str(entry.get("command") or "").strip() + if not command: + raise RuntimeError("the registered ACP entry has an empty command") + executable = Path(command) + if not executable.is_file(): + raise RuntimeError(f"the registered ACP entry command does not exist: {executable}") + if not os.access(executable, os.X_OK): + raise RuntimeError(f"the registered ACP entry command is not executable: {executable}") + + args = [str(item) for item in (entry.get("args") or [])] + env = entry.get("env") or {} + agent_arg = _flag_value(args, "--agent-cmd") + proxy_args = args[:args.index("--agent-cmd")] if "--agent-cmd" in args else args + agent_digest = _normalize_digest(_flag_value(proxy_args, "--agent-digest")) + if not agent_arg: + raise RuntimeError("the registered ACP entry declares no --agent-cmd executable") + if not agent_arg.strip(): + raise RuntimeError("the registered ACP entry has an empty --agent-cmd executable") + if not agent_digest: + raise RuntimeError("the registered ACP entry declares no --agent-digest before --agent-cmd") + agent_executable = Path(agent_arg).expanduser() + if not agent_executable.is_file(): + raise RuntimeError(f"the registered entry's agent binary does not exist: {agent_executable}") + if not os.access(agent_executable, os.X_OK): + raise RuntimeError(f"the registered entry's agent binary is not executable: {agent_executable}") + + proxy_identity = real_agents.executable_identity(executable, framework="code4me2-agent", source="registered_entry") + agent_identity = real_agents.executable_identity(agent_executable, framework="code4me2-agent", source="registered_entry") + if agent_digest and agent_digest != agent_identity.sha256: + raise RuntimeError( + "the registered entry's --agent-digest does not match the agent binary on disk " + f"({agent_digest} != {agent_identity.sha256})" + ) + return { + "registry": str(registry_path), + "command": str(executable), + "args": args, + # Keys only: the entry env may carry a one-time capability value. + "env_keys": sorted(str(key) for key in env), + # Not a secret: the research session this entry reports telemetry under. + "research_session_id": (env or {}).get("CODE4ME_RESEARCH_SESSION_ID"), + "adapter_id": (env or {}).get("CODE4ME_AGENT_ADAPTER_ID"), + "proxy": proxy_identity.to_dict(), + "agent_command": str(agent_executable), + "agent_argv": args[args.index("--agent-cmd") + 1:], + "agent_digest_flag_matches": bool(agent_digest) and agent_digest == agent_identity.sha256, + "agent": agent_identity.to_dict(), + } + + +# --------------------------------------------------------------------------- +# Scoped telemetry +# --------------------------------------------------------------------------- + + +def telemetry_scope_sql( + enrollment_id: str, + emitter_id: str, + started_at: str, + research_session_id: Optional[str] = None, +) -> str: + """Rows for this enrollment emitted by this launch, plus this run's IDE rows. + + Proxy rows are pinned to this launch's unique emitter id and its time + window. IDE rows cannot be pinned in time: the IDE emits its own events + during sign-in and activation, before the harness launches the proxy. They + are pinned to the per-run enrollment and, when ``research_session_id`` is + supplied, to that exact research session (``ide:`` emitters are per + project, not per run); the proxy rows must belong to the same session. + """ + uuid.UUID(enrollment_id) # fail closed on a malformed id before SQL + safe_emitter = emitter_id.replace("'", "") + safe_started = started_at.replace("'", "") + session_filter = "" + if research_session_id: + uuid.UUID(research_session_id) # fail closed on a malformed id before SQL + safe_session = research_session_id.replace("'", "") + session_filter = f"AND research_session_id='{safe_session}' " + return ( + "SELECT emitter_id, source, event_type, count(*) FROM research_event " + f"WHERE enrollment_id='{enrollment_id}' " + f"{session_filter}" + f"AND ((emitter_id='{safe_emitter}' AND occurred_at >= '{safe_started}') " + "OR emitter_id LIKE 'ide:%') " + "GROUP BY 1,2,3;" + ) + + +def parse_telemetry_rows(rows: str) -> List[Tuple[str, str, str, int]]: + parsed: List[Tuple[str, str, str, int]] = [] + for line in rows.splitlines(): + parts = line.split("|") + if len(parts) != 4: + continue + try: + count = int(parts[3]) + except ValueError: + continue + parsed.append((parts[0], parts[1], parts[2], count)) + return parsed + + +def evaluate_telemetry(rows: Sequence[Tuple[str, str, str, int]], emitter_id: str) -> Dict[str, Any]: + """Fail unless this emitter's canonical event projection is complete. + + ``ide`` telemetry from this run's window must also be present. + """ + scoped_counts: Dict[str, int] = {} + sources = set() + for row_emitter, source, event_type, count in rows: + if (row_emitter == emitter_id and source == "acp") or (row_emitter.startswith("ide:") and source == "ide"): + sources.add(source) + if row_emitter == emitter_id and source == "acp": + scoped_counts[event_type] = scoped_counts.get(event_type, 0) + count + missing_sources = sorted(set(REQUIRED_TELEMETRY_SOURCES) - sources) + missing_events = { + event_type: {"required": minimum, "observed": scoped_counts.get(event_type, 0)} + for event_type, minimum in REQUIRED_ACP_EVENT_COUNTS.items() + if scoped_counts.get(event_type, 0) < minimum + } + if missing_sources or missing_events: + raise AssertionError( + "scoped ACP/IDE telemetry incomplete: " + f"missing_sources={missing_sources} missing_events={missing_events}" + ) + return { + "sources": sorted(sources), + "acp_event_counts": dict(sorted(scoped_counts.items())), + "required_event_counts": dict(REQUIRED_ACP_EVENT_COUNTS), + } + + +def wait_for_scoped_telemetry(scenario, enrollment_id: str, emitter_id: str, + started_at: str, research_session_id: Optional[str] = None, + timeout: int = 45) -> Dict[str, Any]: + """Poll the real DB until this invocation's scoped telemetry is complete.""" + deadline = time.monotonic() + timeout + last_error = "" + while time.monotonic() < deadline: + rows = parse_telemetry_rows( + stack.psql(scenario, telemetry_scope_sql(enrollment_id, emitter_id, started_at, + research_session_id)) + ) + try: + return evaluate_telemetry(rows, emitter_id) + except AssertionError as error: + last_error = str(error) + time.sleep(1) + raise AssertionError(last_error or "scoped telemetry did not appear before the deadline") + + +def exercise(scenario, run_path: Path, state: dict, stub) -> dict: + _, registered = registered_entry(run_path) + entry = entry_identity(run_path) + env = {key: value for key, value in os.environ.items() + if not key.startswith(("CODE4ME_", "OPENAI_", "ANTHROPIC_"))} + env.update({str(key): str(value) for key, value in (registered.get("env") or {}).items()}) + env["HOME"] = str(run_path / "ide-home") + emitter_id = "acp-proxy:e2e-" + uuid.uuid4().hex + started_at = datetime.now(timezone.utc).isoformat() + stub_before = stub.request_count() if stub is not None else 0 + with (run_path / "acp-runtime.log").open("w") as log: + child = subprocess.Popen([entry["command"], "--emitter-id", emitter_id, *entry["args"]], + cwd=run_path / "ui-project", env=env, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=log, text=True, bufsize=1, start_new_session=True) + try: + client = AcpClient(child) + initialized = client.request("initialize", { + "protocolVersion": 1, "clientCapabilities": {}, + "clientInfo": {"name": "code4me-e2e", "version": "1"}, + }) + protocol_version = initialized.get("protocolVersion") + if type(protocol_version) is not int or protocol_version != 1: + raise AssertionError(f"ACP initialize returned no usable protocolVersion: {protocol_version!r}") + session = client.request("session/new", {"cwd": str(run_path / "ui-project"), "mcpServers": []}) + if not session.get("sessionId"): + raise RuntimeError("ACP session/new returned no sessionId") + outcome = client.request("session/prompt", { + "sessionId": session["sessionId"], + "prompt": [{"type": "text", "text": scenario.message.prompt}], + }, scenario.timeouts.step_seconds) + expected = scenario.message.expected_substring or stub.token + if expected not in "".join(client.chunks): + raise AssertionError("ACP answer did not contain the provider's deterministic token") + if outcome.get("stopReason") != "end_turn": + raise AssertionError(f"Unexpected ACP stop reason: {outcome.get('stopReason')}") + # Per-turn proof: the first prompt must be exactly one new provider + # request. A request left by an earlier UI/prefix run cannot satisfy it. + stub_after = stub.request_count() if stub is not None else 0 + if stub is None or stub_after - stub_before != 1: + raise AssertionError( + "the first ACP prompt did not produce exactly one provider request " + f"(before={stub_before}, after={stub_after})" + ) + finally: + # The proxy forwards the response before observing it. Give EOF + # handling time to flush observations into the IDE's durable spool. + # Killing the process group immediately can lose that final event. + try: + child.stdin.close() + child.wait(timeout=15) + except (OSError, subprocess.TimeoutExpired): + pass + finally: + process.stop(child) + child.stdout.close() + # Verify plugin/proxy telemetry reached the real DB through the IDE spool. + # Require this proxy launch and this run's IDE window, never events left by + # a previous run/fixture. + enrollment = str(state["enrollment_id"]) + # The registered entry names the IDE's research session; the HTTP workflow + # state only knows one after its own acp_prepare step. + research_session_id = entry.get("research_session_id") or state.get("research_session_id") + telemetry = wait_for_scoped_telemetry(scenario, enrollment, emitter_id, started_at, + research_session_id) + telemetry["research_session_scoped"] = bool(research_session_id) + return {"protocol_version": protocol_version, + "acp_methods": list(client.methods), + "required_method_event_types": dict(ACP_METHOD_EVENT_TYPES), + "entry": entry, + "stub_request_delta": stub_after - stub_before, + "telemetry_window_started_at": started_at, + "answer_contains_expected": True, + "telemetry": telemetry} diff --git a/e2e/code4me_e2e/acp_probe.py b/e2e/code4me_e2e/acp_probe.py new file mode 100644 index 0000000..68a5eda --- /dev/null +++ b/e2e/code4me_e2e/acp_probe.py @@ -0,0 +1,408 @@ +"""Strict ACP stdio probe for a real participant-installed (BYOA) agent. + +The managed-proxy path uses :mod:`code4me_e2e.acp` with a deterministic stub +provider. This module instead launches the *real* Goose/Codex executable the +participant installed, over stdio, and exercises the ACP handshake the IDE would +perform: ``initialize`` -> ``session/new`` -> ``session/prompt``. + +Every step is validated, not assumed: a non-empty executable entry, an +``initialize`` result with a protocol version and an agent-capabilities +mapping, a non-empty session id, and a completed first prompt. A missing +binary, missing authentication, exhausted quota or protocol failure is a typed +``BLOCKED`` result and can never be reported as a pass. +""" + +from __future__ import annotations + +import json +import queue +import subprocess +import threading +import time +from pathlib import Path +from typing import Callable, Any, Dict, List, Mapping, Optional, Sequence + +from . import process +from .config import AgentProbeReason +from .real_agents import ( + AgentIdentity, + ProbeResult, + build_env, + classify_failure, + isolated_home, + plan_argv, + probe_identity, + redact_failure_text, +) + +__all__ = ["DEFAULT_PROMPT", "AcpProbeError", "AcpRemoteError", "StdioAcpSession", "run_probe"] + +#: The deterministic first turn. No user content is ever sent to a real agent. +DEFAULT_PROMPT = "Say hello in one short sentence." + +#: ACP JSON-RPC error code for unimplemented client methods. The probe provides +#: no filesystem/terminal host methods, so those calls fail closed. +_METHOD_NOT_FOUND = -32601 + +#: Bounded diagnostic transcript length (lines). +_TRANSCRIPT_LIMIT = 200 + + +class AcpProbeError(RuntimeError): + """The probe could not complete a step (timeout, closed pipe, bad shape).""" + + def __init__(self, message: str, *, reason: Optional[AgentProbeReason] = None): + super().__init__(message) + self.reason = reason + + +class AcpRemoteError(AcpProbeError): + """The agent answered a request with a JSON-RPC error.""" + + def __init__(self, method: str, code: Any, message: str): + super().__init__(f"ACP {method} failed (code {code}): {message}") + self.method = method + self.code = code + self.remote_message = message + + +class StdioAcpSession: + """Minimal ACP client over stdio. No host tools are executed.""" + + def __init__(self, child: subprocess.Popen, *, timeout: float = 120.0): + self.child = child + self.timeout = timeout + self.messages: "queue.Queue[Any]" = queue.Queue() + self.chunks: List[str] = [] + self.methods: List[str] = [] + self.transcript: List[str] = [] + self.sequence = 0 + threading.Thread(target=self._read, daemon=True).start() + + # -- stdio ------------------------------------------------------------- + + def _read(self) -> None: + try: + for line in self.child.stdout: + self._remember(line) + try: + self.messages.put(json.loads(line)) + except ValueError: + continue + except Exception as error: # noqa: BLE001 - surfaced to the requester + self.messages.put(error) + finally: + self.messages.put(EOFError("ACP process closed stdout")) + + def _remember(self, line: str) -> None: + self.transcript.append(line.strip()[:500]) + del self.transcript[:-_TRANSCRIPT_LIMIT] + + def _send(self, payload: Dict[str, Any]) -> None: + self.child.stdin.write(json.dumps({"jsonrpc": "2.0", **payload}) + "\n") + self.child.stdin.flush() + + # -- protocol ---------------------------------------------------------- + + def request(self, method: str, params: Dict[str, Any], timeout: Optional[float] = None) -> Dict[str, Any]: + """Send one request and wait for its result, answering host calls.""" + self.sequence += 1 + request_id = self.sequence + self.methods.append(method) + self._send({"id": request_id, "method": method, "params": params}) + deadline = time.monotonic() + (timeout if timeout is not None else self.timeout) + while time.monotonic() < deadline: + try: + message = self.messages.get(timeout=max(0.01, deadline - time.monotonic())) + except queue.Empty: + break + if isinstance(message, Exception): + raise AcpProbeError(str(message)) + if message.get("method") == "session/update": + update = message.get("params", {}).get("update", {}) + if update.get("sessionUpdate") == "agent_message_chunk": + text = update.get("content", {}).get("text", "") + if text: + self.chunks.append(text) + elif "method" in message and "id" in message: + self._answer_host_call(message) + elif message.get("id") == request_id: + if "error" in message: + error = message["error"] or {} + raise AcpRemoteError(method, error.get("code"), str(error.get("message") or "")) + result = message.get("result", {}) + if not isinstance(result, dict): + raise AcpProbeError(f"ACP {method} returned a non-object result") + return result + raise AcpProbeError( + f"ACP {method} did not finish within {int(self.timeout)}s", + reason=AgentProbeReason.PROTOCOL, + ) + + def _answer_host_call(self, message: Dict[str, Any]) -> None: + """Fail-closed host responses: no fs/terminal access, no permission.""" + method = message.get("method") + if method == "session/request_permission": + self._send({"id": message["id"], "result": {"outcome": {"outcome": "cancelled"}}}) + return + self._send({ + "id": message["id"], + "error": {"code": _METHOD_NOT_FOUND, "message": "Unsupported host method"}, + }) + + def drain(self, timeout: float = 0.5) -> None: + """Collect trailing notifications without waiting on a response.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + message = self.messages.get(timeout=max(0.01, deadline - time.monotonic())) + except queue.Empty: + return + if isinstance(message, Exception): + return + if message.get("method") == "session/update": + update = message.get("params", {}).get("update", {}) + if update.get("sessionUpdate") == "agent_message_chunk": + text = update.get("content", {}).get("text", "") + if text: + self.chunks.append(text) + + def close(self) -> None: + try: + if self.child.stdin and not self.child.stdin.closed: + self.child.stdin.close() + except OSError: + pass + try: + self.child.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + process.stop(self.child) + # The reader thread has reached EOF (or is about to); releasing the + # stdout pipe keeps a finished probe from leaking a file object until + # garbage collection. A concurrent read error is already handled. + if self.child.stdout is not None: + try: + self.child.stdout.close() + except (OSError, ValueError): + pass + + +def _read_tail(path: Path, limit: int = 2000) -> str: + try: + with path.open("r", encoding="utf-8", errors="replace") as handle: + text = handle.read() + except OSError: + return "" + return text[-limit:] + + +def _diagnostic(session: Optional[StdioAcpSession], stderr_path: Path, error: Exception) -> str: + parts = [str(error)] + if session is not None and session.transcript: + parts.append("transcript: " + " | ".join(session.transcript[-6:])) + tail = _read_tail(stderr_path) + if tail.strip(): + parts.append("stderr: " + tail) + return redact_failure_text(" :: ".join(parts)) + + +def _blocked_from_failure( + framework: str, + identity: AgentIdentity, + session: Optional[StdioAcpSession], + stderr_path: Path, + error: Exception, + *, + checks: Sequence[Mapping[str, Any]], +) -> ProbeResult: + diagnostic = _diagnostic(session, stderr_path, error) + reason = classify_failure(diagnostic) + return ProbeResult.blocked( + framework, + reason, + diagnostic or str(error), + identity=identity, + checks=checks, + protocol={"executable": identity.path, "methods": list(session.methods) if session else []}, + ) + + +def run_probe( + framework: str, + *, + run_dir: Path, + executable: Optional[str] = None, + command: Optional[str] = None, + package: Optional[str] = None, + argv: Optional[Sequence[str]] = None, + home: Optional[str] = None, + prompt: str = DEFAULT_PROMPT, + timeout: int = 120, + env_extra: Optional[Mapping[str, str]] = None, + base_env: Optional[Mapping[str, str]] = None, + expected_substring: Optional[str] = None, + prepare_home: Optional[Callable[[Path], None]] = None, +) -> ProbeResult: + """Run the real ACP handshake against the host-installed agent. + + Returns a typed :class:`ProbeResult`; it never raises for a missing or + unusable prerequisite, so a caller can render the blocked reason. + + ``prepare_home`` runs once the empty isolated home exists and before the + environment is built, so a caller can seed it (for example with a + deliberately poisoned agent config the launch must override). + """ + run_path = Path(run_dir) + run_path.mkdir(parents=True, exist_ok=True) + identity_result = probe_identity( + framework, + executable=executable, + command=command, + package=package, + env=base_env, + ) + if not identity_result.passed or identity_result.identity is None: + return identity_result + identity = identity_result.identity + + plan = plan_argv(framework, identity, argv) + if plan.blocked: + return ProbeResult.blocked( + framework, + plan.blocked_reason or AgentProbeReason.PROTOCOL, + plan.blocked_detail, + identity=identity, + protocol={"argv": plan.argv}, + ) + + home_path = isolated_home(run_path, framework, home) + if prepare_home is not None: + prepare_home(home_path) + workspace = run_path / "agent-workspace" / framework + workspace.mkdir(parents=True, exist_ok=True) + env = build_env(home_path, base_env=base_env, extra=env_extra) + stderr_path = run_path / f"agent-{framework}.stderr.log" + + checks: List[Dict[str, Any]] = [] + session: Optional[StdioAcpSession] = None + with stderr_path.open("w", encoding="utf-8") as stderr_log: + try: + child = subprocess.Popen( + plan.argv, + cwd=str(workspace), + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=stderr_log, + text=True, + bufsize=1, + start_new_session=True, + ) + except OSError as error: + return ProbeResult.blocked( + framework, + AgentProbeReason.MISSING_BINARY, + f"could not execute {identity.path!r}: {error}", + identity=identity, + protocol={"argv": plan.argv}, + checks=checks, + ) + try: + session = StdioAcpSession(child, timeout=float(timeout)) + + initialized = session.request("initialize", { + "protocolVersion": 1, + "clientCapabilities": {}, + "clientInfo": {"name": "code4me-e2e-agent-probe", "version": "1"}, + }) + protocol_version = initialized.get("protocolVersion") + capabilities = initialized.get("agentCapabilities") + if capabilities is None: + capabilities = initialized.get("capabilities") + if type(protocol_version) is not int or protocol_version != 1: + raise AcpProbeError( + f"initialize returned no usable protocolVersion: {protocol_version!r}", + reason=AgentProbeReason.PROTOCOL, + ) + if not isinstance(capabilities, Mapping): + raise AcpProbeError( + "initialize returned no agentCapabilities object", + reason=AgentProbeReason.PROTOCOL, + ) + checks.append({ + "check": "initialize", + "protocol_version": protocol_version, + "capability_keys": sorted(str(key) for key in capabilities)[:20], + }) + + session_result = session.request("session/new", { + "cwd": str(workspace), + "mcpServers": [], + }) + session_id = session_result.get("sessionId") + if not isinstance(session_id, str) or not session_id.strip(): + raise AcpProbeError( + "session/new returned no non-empty sessionId", + reason=AgentProbeReason.PROTOCOL, + ) + checks.append({"check": "session/new", "session_id_present": True}) + + prompt_result = session.request("session/prompt", { + "sessionId": session_id, + "prompt": [{"type": "text", "text": prompt}], + }, timeout=float(timeout)) + session.drain(timeout=1.0) + stop_reason = prompt_result.get("stopReason") + if not isinstance(stop_reason, str) or not stop_reason: + raise AcpProbeError( + "session/prompt returned no stopReason", + reason=AgentProbeReason.PROTOCOL, + ) + if stop_reason != "end_turn": + raise AcpProbeError( + f"session/prompt stopped with {stop_reason!r} instead of end_turn", + reason=AgentProbeReason.PROTOCOL, + ) + answer = "".join(session.chunks) + if not answer.strip(): + raise AcpProbeError( + "session/prompt completed without an assistant message", + reason=AgentProbeReason.PROTOCOL, + ) + if expected_substring is not None and expected_substring not in answer: + raise AcpProbeError( + "session/prompt did not return the expected provider answer (an error message is not a model response)", + reason=AgentProbeReason.PROTOCOL, + ) + checks.append({ + "check": "session/prompt", + "stop_reason": stop_reason, + # Content is never stored: only the fact and size of the answer. + "assistant_message_chars": len(answer), + }) + except Exception as error: # noqa: BLE001 - every failure is a typed block + return _blocked_from_failure(framework, identity, session, stderr_path, error, checks=checks) + finally: + if session is not None: + session.close() + else: + process.stop(child) + + return ProbeResult.ok( + framework, + identity, + checks=checks, + protocol={ + "argv": plan.argv, + "home": str(home_path), + "env_keys": sorted( + key for key in env if key in ( + "HOME", "CODEX_HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME", + "XDG_CACHE_HOME", "XDG_STATE_HOME", "PATH", "NO_COLOR", "TERM", + ) + ), + "methods": list(session.methods) if session else [], + "stderr_log": str(stderr_path), + }, + ) diff --git a/e2e/code4me_e2e/agent_provider.py b/e2e/code4me_e2e/agent_provider.py new file mode 100644 index 0000000..d75fde3 --- /dev/null +++ b/e2e/code4me_e2e/agent_provider.py @@ -0,0 +1,291 @@ +"""Loopback OpenAI transport for native Goose and Codex ACP probes. + +The provider implements only the two wire protocols exercised by the native +agents: Chat Completions for Goose and Responses for the vendored Codex ACP +adapter. It deliberately keeps request receipts to metadata, never request +bodies or authentication headers, so a test can prove a fresh outbound call +without preserving prompt or credential material. +""" + +from __future__ import annotations + +import json +import threading +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, List, Optional + +__all__ = ["AgentProvider", "NativeAgentProvider"] + + +#: The research inference gateway's Chat Completions path (no leading slash +#: on the Goose side; the provider serves it with one). +GATEWAY_CHAT_ROUTE = "/api/research/inference/v1/chat/completions" + + +class AgentProvider: + """A deterministic, ephemeral-port OpenAI-compatible loopback provider. + + ``chat_routes`` are the Chat Completions paths it answers (the classic + ``/v1/chat/completions`` and the research gateway path by default). + ``expected_bearer`` lets a receipt record whether the ``Authorization`` + header matched (equality only; the value is never stored). With + ``quota_exhausted`` every chat call is refused with the research gateway's + ``402 quota_exhausted`` body, which is how a used-up participant budget + looks to the agent. + """ + + def __init__( + self, + token: Optional[str] = None, + *, + chat_routes: Optional[List[str]] = None, + expected_bearer: Optional[str] = None, + quota_exhausted: bool = False, + ) -> None: + self.token = token or f"E2E_NATIVE_ANSWER_{uuid.uuid4().hex}" + self.chat_routes = list(chat_routes or ["/v1/chat/completions", GATEWAY_CHAT_ROUTE]) + self.expected_bearer = expected_bearer + self.quota_exhausted = bool(quota_exhausted) + self.requests: List[Dict[str, Any]] = [] + self.port: Optional[int] = None + self._server: Optional[ThreadingHTTPServer] = None + self._thread: Optional[threading.Thread] = None + self._lock = threading.Lock() + self._request_count = 0 + + def start(self, port: int = 0) -> int: + """Start on ``127.0.0.1`` and return the selected port.""" + if self._server is not None: + return self.port or 0 + server = ThreadingHTTPServer(("127.0.0.1", port), _make_handler(self)) + server.daemon_threads = True + self._server = server + self.port = int(server.server_address[1]) + self._thread = threading.Thread( + target=server.serve_forever, + name="code4me-e2e-native-provider", + daemon=True, + ) + self._thread.start() + return self.port + + def stop(self) -> None: + """Stop the serving thread cleanly; repeated calls are harmless.""" + server, thread = self._server, self._thread + self._server = None + self._thread = None + if server is not None: + server.shutdown() + server.server_close() + if thread is not None: + thread.join(timeout=5) + + @property + def base_url(self) -> str: + if self.port is None: + raise RuntimeError("AgentProvider has not been started") + return f"http://127.0.0.1:{self.port}" + + def record( + self, *, route: str, model: Any, stream: bool, auth_matched: Optional[bool] = None + ) -> None: + """Add a metadata-only, monotonic request receipt (never a header value).""" + with self._lock: + self._request_count += 1 + receipt = { + "route": route, + "model": str(model) if model is not None else None, + "stream": bool(stream), + "count": self._request_count, + # Kept for drop-in compatibility with ``StubProvider`` callers. + "receipt": self._request_count, + # None when no expected bearer was configured. + "auth_matched": auth_matched, + } + self.requests.append(receipt) + + def quota_refusal(self) -> Dict[str, Any]: + """The research gateway's refusal body for a used-up budget.""" + return { + "error": { + "message": ( + "Your study's AI budget is used up (available $0.00; this request " + "needs at least $0.01). Ask the study team for a top-up." + ), + "type": "insufficient_quota", + "code": "quota_exhausted", + } + } + + def request_count(self) -> int: + with self._lock: + return self._request_count + + def requests_since(self, receipt: int) -> List[Dict[str, Any]]: + with self._lock: + return [dict(item) for item in self.requests if item["count"] > receipt] + + def chat_completion(self, model: Any) -> Dict[str, Any]: + return { + "id": "chatcmpl-e2e-native", + "object": "chat.completion", + "created": 0, + "model": str(model or "e2e-stub-model"), + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": self.token}, + "finish_reason": "stop", + }], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + def response(self, model: Any) -> Dict[str, Any]: + message = { + "id": "msg_e2e_native", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": self.token, "annotations": []}], + } + return { + "id": "resp_e2e_native", + "object": "response", + "status": "completed", + "model": str(model or "e2e-stub-model"), + "output": [message], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + + +# A descriptive alias helps callers distinguish this native-agent fixture from +# the existing backend relay ``StubProvider`` while retaining one implementation. +NativeAgentProvider = AgentProvider + + +def _make_handler(provider: AgentProvider): + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args: Any) -> None: + return + + def _send_json(self, status: int, payload: Dict[str, Any]) -> None: + encoded = json.dumps(payload, separators=(",", ":")).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(encoded) + + def _send_sse(self, events: List[tuple[str, Any]]) -> None: + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.end_headers() + for event, payload in events: + self.wfile.write(f"event: {event}\n".encode("utf-8")) + if payload == "[DONE]": + self.wfile.write(b"data: [DONE]\n\n") + else: + data = json.dumps(payload, separators=(",", ":")) + self.wfile.write(f"data: {data}\n\n".encode("utf-8")) + self.wfile.flush() + + def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + if self.path.split("?", 1)[0] == "/health": + self._send_json(200, {"status": "ok"}) + return + self._send_json(404, {"error": {"message": "unsupported route", "type": "invalid_request_error"}}) + + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + route = self.path.split("?", 1)[0] + if route not in (*provider.chat_routes, "/v1/responses"): + self._send_json(404, {"error": {"message": "unsupported route", "type": "invalid_request_error"}}) + return + auth_matched: Optional[bool] = None + if provider.expected_bearer is not None: + header = self.headers.get("Authorization") or "" + auth_matched = header == f"Bearer {provider.expected_bearer}" + try: + length = int(self.headers.get("Content-Length") or "0") + if length < 0: + raise ValueError + payload = json.loads(self.rfile.read(length).decode("utf-8")) if length else {} + except (UnicodeDecodeError, ValueError, json.JSONDecodeError): + self._send_json(400, {"error": {"message": "invalid JSON", "type": "invalid_request_error"}}) + return + if not isinstance(payload, dict): + self._send_json(400, {"error": {"message": "request must be an object", "type": "invalid_request_error"}}) + return + + model, stream = payload.get("model"), bool(payload.get("stream")) + # Do not retain ``payload``, headers, or any prompt/auth values. + provider.record(route=route, model=model, stream=stream, auth_matched=auth_matched) + if route in provider.chat_routes: + if provider.quota_exhausted: + # The research gateway refuses before anything reaches a + # provider; 402 is deliberately not a retried status. + self._send_json(402, provider.quota_refusal()) + return + self._chat(model, stream) + else: + self._responses(model, stream) + + def _chat(self, model: Any, stream: bool) -> None: + completion = provider.chat_completion(model) + if not stream: + self._send_json(200, completion) + return + chunk = { + "id": completion["id"], "object": "chat.completion.chunk", "created": 0, + "model": completion["model"], + "choices": [{"index": 0, "delta": {"role": "assistant", "content": provider.token}, "finish_reason": None}], + } + done = {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]} + self._send_sse([("chat.completion.chunk", chunk), ("chat.completion.chunk", done), ("done", "[DONE]")]) + + def _responses(self, model: Any, stream: bool) -> None: + response = provider.response(model) + if not stream: + self._send_json(200, response) + return + started = {**response, "status": "in_progress", "output": []} + in_progress_item = { + "id": "msg_e2e_native", "type": "message", "status": "in_progress", + "role": "assistant", "content": [], + } + completed_item = response["output"][0] + self._send_sse([ + ("response.created", {"type": "response.created", "response": started}), + ("response.in_progress", {"type": "response.in_progress", "response": started}), + ("response.output_item.added", { + "type": "response.output_item.added", "output_index": 0, "item": in_progress_item, + }), + ("response.content_part.added", { + "type": "response.content_part.added", "item_id": "msg_e2e_native", + "output_index": 0, "content_index": 0, + "part": {"type": "output_text", "text": "", "annotations": []}, + }), + ("response.output_text.delta", { + "type": "response.output_text.delta", "item_id": "msg_e2e_native", + "output_index": 0, "content_index": 0, "delta": provider.token, + }), + ("response.output_text.done", { + "type": "response.output_text.done", "item_id": "msg_e2e_native", + "output_index": 0, "content_index": 0, "text": provider.token, + }), + ("response.content_part.done", { + "type": "response.content_part.done", "item_id": "msg_e2e_native", + "output_index": 0, "content_index": 0, + "part": completed_item["content"][0], + }), + ("response.output_item.done", { + "type": "response.output_item.done", "output_index": 0, "item": completed_item, + }), + ("response.completed", {"type": "response.completed", "response": response}), + ]) + + return Handler diff --git a/e2e/code4me_e2e/browser.py b/e2e/code4me_e2e/browser.py new file mode 100644 index 0000000..d644009 --- /dev/null +++ b/e2e/code4me_e2e/browser.py @@ -0,0 +1,435 @@ +"""Run the website scenarios with isolated state and persisted-outcome proof. + +The browser layer is independent of the IDE layer. It provisions the study it +needs in its own ``/browser-state`` directory, so the IDE layers' +enrollments/revocation state in ``/state.json`` is never overwritten, and +it is attempted even when the IDE layer fails. + +Website success is only reported when the exact scenario inventory ran once +each and all checks passed, the results carry no join code/credential, and the +persisted read models (edited metadata, stopped state, clone identity/profile +selection, enrollment/assignment coverage) still show the same outcomes. +""" +from __future__ import annotations + +import contextlib +import copy +import io +import json +import os +import shutil +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +from dataclasses import asdict +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence + +from . import process, report, stack, workflow +from .paths import BROWSER_DIR, require_workspace +from .steps import STEP_ORDER, StepResult + +BROWSER_STEP_ID = "browser_test" + +#: Run-state directory (below the layer's run directory) owned by the browser +#: layer alone; the IDE layers keep using ``/state.json``. +BROWSER_STATE_DIRNAME = "browser-state" + +#: Every check the website suite must execute exactly once with status PASS. +#: ``scenarios.py`` runs under the browser Python (Playwright), which the +#: harness interpreter may not have, so the inventory lives here, is passed to +#: the child through the environment, and is re-validated by the parent. +EXPECTED_SCENARIO_IDS: tuple[str, ...] = ( + "A1", "A2", "A3", "A4", "A5", "A6", "A7", "A8", "A9", "A10", "A11", "A12", + "A13", + "B1", "B2", "B3", "B4", "B5", "B6", "B7", "B8", + "C1", "C2", "C3", "C4", "C5", "C6", + "D1", "D2", "D3", "D4", "D5", "D6", + "E1", "E2", "E3", "E4", "E5", "E6", "E7", +) + +#: Identity the child must return so the parent can re-read the persisted +#: models itself. Values are ids/names only; never a join code or credential. +CONTEXT_KEYS: tuple[str, ...] = ( + "study_id", + "edited_name", + "edited_description", + "clone_study_id", + "clone_name", + "profile_id", + "enrollment_id", + "assignment_id", +) + + +class PrerequisiteError(RuntimeError): + """A missing prerequisite (Playwright, Node, reachable stack). + + Reported as BLOCKED with a reason, never as a pass and never with backend + logs captured from a stack that was never reached. + """ + + +def state_directory(run_path: Path, state_dir: Optional[Path] = None) -> Path: + """The browser layer's own run-state directory (never the layer run dir).""" + base = Path(run_path).resolve() + return Path(state_dir).resolve() if state_dir else base / BROWSER_STATE_DIRNAME + + +# --------------------------------------------------------------------------- +# Evidence validation (unit-testable without a live stack) +# --------------------------------------------------------------------------- + + +def validate_scenario_results( + payload: Any, *, expected: Sequence[str] = EXPECTED_SCENARIO_IDS +) -> List[Dict[str, Any]]: + """Validate the exact scenario inventory; return the normalized steps. + + Missing, duplicated, blocked or unexpected ids can never pass: every + expected id must appear exactly once, with status PASS and no other id. + """ + expected_ids = list(expected) + steps = payload.get("steps") if isinstance(payload, dict) else None + if not isinstance(steps, list) or not steps: + raise RuntimeError("website evidence carried no steps array") + ids: List[str] = [] + for step in steps: + if not isinstance(step, dict) or not str(step.get("id") or "").strip(): + raise RuntimeError("website evidence contains a step without an id") + ids.append(str(step["id"])) + duplicates = sorted({step_id for step_id in ids if ids.count(step_id) > 1}) + missing = [step_id for step_id in expected_ids if step_id not in ids] + unexpected = [step_id for step_id in ids if step_id not in expected_ids] + not_passing = [ + f"{step.get('id')}:{step.get('status')}" + for step in steps + if str(step.get("status", "")).upper() != "PASS" + ] + problems: List[str] = [] + if missing: + problems.append("missing " + ",".join(missing)) + if duplicates: + problems.append("duplicated " + ",".join(duplicates)) + if unexpected: + problems.append("unexpected " + ",".join(unexpected)) + if len(ids) != len(expected_ids): + problems.append(f"expected {len(expected_ids)} checks, got {len(ids)}") + if not_passing: + problems.append("not passing " + ",".join(not_passing)) + if problems: + raise RuntimeError("website scenario evidence invalid: " + "; ".join(problems)) + return steps + + +def validate_context(payload: Any, *, keys: Sequence[str] = CONTEXT_KEYS) -> Dict[str, str]: + """The child's identity hand-off for the parent's persisted re-reads.""" + context = payload.get("context") if isinstance(payload, dict) else None + if not isinstance(context, dict): + raise RuntimeError("website evidence carried no context object") + missing = [key for key in keys if not str(context.get(key) or "").strip()] + if missing: + raise RuntimeError("website evidence context is missing " + ", ".join(missing)) + resolved = {key: str(context[key]) for key in keys} + if resolved["study_id"] == resolved["clone_study_id"]: + raise RuntimeError("website evidence reports the clone with the source study id") + return resolved + + +def assert_no_secret_material(text: str, *, secrets: Sequence[str]) -> None: + """Refuse to pass when evidence contains a join code or credential value.""" + for secret in secrets: + if secret and secret in text: + raise RuntimeError( + "website evidence contains private material (join code or " + "credential); refusing to pass" + ) + + +# --------------------------------------------------------------------------- +# Persisted read-model verification (independent of the child's own checks) +# --------------------------------------------------------------------------- + + +def _api_get( + base_url: str, path: str, cookies: Dict[str, str], timeout: float = 20.0 +) -> Dict[str, Any]: + request = urllib.request.Request( + base_url.rstrip("/") + path, + headers={ + "Accept": "application/json", + "Cookie": "; ".join(f"{key}={value}" for key, value in cookies.items()), + }, + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return {"status": response.status, "payload": json.loads(response.read() or b"{}")} + except urllib.error.HTTPError as error: + body = error.read() + try: + payload = json.loads(body or b"{}") + except json.JSONDecodeError: + payload = {} + return {"status": error.code, "payload": payload} + + +def login_cookies(scenario, role: str = "researcher") -> Dict[str, str]: + """A fresh session for ``role``: the browser's own sign-in may end older ones.""" + from .http import HttpClient + + account = getattr(scenario, role) + client = HttpClient(scenario.base_url or "", label=f"browser-verify-{role}", timeout=20.0) + response = client.post("/api/user/authenticate", {"email": account.email, "password": account.password}) + if response.status != 200: + raise RuntimeError(f"signing in as the {role} for the persisted re-read failed (HTTP {response.status})") + return client.export_cookies() + + +def verify_persisted_outcomes(scenario, state: Dict[str, Any], context: Dict[str, str], + *, cookies: Optional[Dict[str, str]] = None) -> Dict[str, bool]: + """Re-read the real API models and require the persisted browser outcomes. + + This never trusts the child's report: it fetches the study, clone and + participant-coverage read models as the researcher, with ``cookies`` from a + fresh sign-in (the runner passes them) or the session the browser setup + workflow persisted. + """ + cookies = dict(cookies or (state.get("cookies") or {}).get("researcher") or {}) + if not cookies: + raise RuntimeError("no researcher session in the browser run state; cannot re-read persisted outcomes") + base_url = scenario.base_url or "" + + original = _api_get(base_url, f"/api/research/studies/{context['study_id']}", cookies) + study = (original.get("payload") or {}).get("study") or {} + clone = _api_get(base_url, f"/api/research/studies/{context['clone_study_id']}", cookies) + clone_study = (clone.get("payload") or {}).get("study") or {} + clone_profiles = [ + str(item.get("profile_id")) for item in (clone_study.get("profile_selections") or []) + ] + clone_capabilities = clone_study.get("lifecycle_capabilities") or {} + coverage = _api_get( + base_url, + f"/api/research/operations/participants/coverage?study_id={context['study_id']}", + cookies, + ) + coverage_data = coverage.get("payload") or {} + rows = [ + row + for row in (coverage_data.get("participants") or []) + if str(row.get("enrollment_id")) == context["enrollment_id"] + ] + assignment = (rows[0].get("assignment") or {}) if rows else {} + + checks = { + "edited_metadata_persisted": ( + original.get("status") == 200 + and study.get("name") == context["edited_name"] + and study.get("description") == context["edited_description"] + and bool(study.get("consent_locked_at")) + ), + "stopped_state_retained": ( + study.get("research_status") == "STUDY_STOPPED" + and study.get("enrollment_count") == 1 + and study.get("active_enrollment_count") == 0 + and study.get("assignment_count") == 1 + ), + "clone_identity_profiles_status": ( + clone.get("status") == 200 + and context["clone_study_id"] != context["study_id"] + and clone_study.get("name") == context["clone_name"] + and clone_study.get("description") == context["edited_description"] + and clone_study.get("research_status") == "DRAFT" + and context["profile_id"] in clone_profiles + and clone_study.get("enrollment_count") == 0 + and clone_study.get("assignment_count") == 0 + and bool(clone_capabilities.get("joinable")) + ), + "enrollment_assignment_coverage": ( + coverage.get("status") == 200 + and coverage_data.get("participant_count") == 1 + and len(rows) == 1 + and rows[0].get("status") == "STUDY_STOPPED" + and str(assignment.get("assignment_id")) == context["assignment_id"] + and str(assignment.get("agent_profile_id")) == context["profile_id"] + ), + } + failed = [name for name, passed in checks.items() if not passed] + if failed: + raise RuntimeError( + "persisted browser outcomes are not visible in the read models: " + ", ".join(failed) + ) + return checks + + +# --------------------------------------------------------------------------- +# Runner +# --------------------------------------------------------------------------- + + +def _require_browser_prerequisites() -> tuple[str, str, str]: + """Resolve the browser Python and Node tooling, or raise BLOCKED.""" + from .prereqs import browser_python + + python, _source = browser_python() + try: + probe = subprocess.run( + [python, "-c", "import playwright.sync_api"], capture_output=True, timeout=30 + ) + except (OSError, subprocess.TimeoutExpired) as error: + raise PrerequisiteError( + f"the Playwright probe could not run ({error}); run `python3 -m code4me_e2e setup " + "--layer browser` or set CODE4ME_E2E_BROWSER_PYTHON (see e2e/README.md)" + ) from error + if probe.returncode: + raise PrerequisiteError( + f"Playwright is not importable by {python}; run `python3 -m code4me_e2e setup " + "--layer browser` or set CODE4ME_E2E_BROWSER_PYTHON (see e2e/README.md)" + ) + node, npm = shutil.which("node"), shutil.which("npm") + if not node or not npm: + raise PrerequisiteError("Node.js and npm must be on PATH for the website scenarios") + return python, node, npm + + +def _bootstrap_blocked_step(state_path: Path) -> Optional[str]: + """The workflow step that blocked the browser bootstrap, if any.""" + try: + built = json.loads((state_path / "report.json").read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(built, dict): + return None + return built.get("blocked_by") or built.get("failed_step") + + +def _workflow_touched_stack(state_path: Path) -> bool: + """True when the bootstrap workflow actually wrote run state or a report.""" + return (state_path / "state.json").is_file() or (state_path / "report.json").is_file() + + +def run(scenario, run_path, *, state_dir: Optional[Path] = None) -> int: + started = time.monotonic() + run_path = Path(run_path) + state_path = state_directory(run_path, state_dir) + child = None + status, details = "FAIL", {} + config_path = run_path / "browser-config.json" + try: + python, node, npm = _require_browser_prerequisites() + prefix = STEP_ORDER[: STEP_ORDER.index("create_study") + 1] + # The layer seeds its own accounts, profile and study: never mutate the + # scenario the other layers of this gate share. + scenario = copy.deepcopy(scenario) + scenario.study.name = f"{scenario.study.name} [{run_path.name}]" + scenario.agent.profile_name = f"e2e-browser-{run_path.name}" + state_path.mkdir(parents=True, exist_ok=True) + try: + with contextlib.redirect_stdout(io.StringIO()): + rc = workflow.run_workflow( + scenario, only=",".join(prefix), run_dir=str(state_path), keep_stack=True + ) + except stack.StackError as error: + raise PrerequisiteError(f"the disposable stack is unavailable: {error}") from error + if rc: + if _bootstrap_blocked_step(state_path) == "doctor": + raise PrerequisiteError( + "the browser layer's backend is not reachable or schema-ready; " + "see browser-state/report.json" + ) + raise RuntimeError("Browser setup failed; see browser-state/report.json") + state = report.read_state(state_path) + workflow.restore_accounts(scenario, state) + root = require_workspace() + website = root / "code4me2-server" / "src/website" + if not (website / "node_modules").is_dir(): + if process.run([npm, "ci"], cwd=website, log_path=run_path / "browser-install.log"): + raise RuntimeError("Website dependency installation failed; see browser-install.log") + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + url = f"http://127.0.0.1:{port}" + web_build = run_path / "website" + # Pin every API call to our same-origin proxy, including legacy login + # code that concatenates host and port. Never inherit production URLs. + build_env = {**os.environ, "CI": "false", "BUILD_PATH": str(web_build), + "REACT_APP_BACKEND_HOST": "http://127.0.0.1", "REACT_APP_BACKEND_PORT": str(port)} + if process.run([npm, "run", "build"], cwd=website, env=build_env, log_path=run_path / "browser-build.log"): + raise RuntimeError("Website build failed; see browser-build.log") + config = {"base_url": scenario.base_url, "ui_url": url, + "config_id": state["config_id"], "study_name": scenario.study.name, + "profile_name": scenario.agent.profile_name, + "researcher": asdict(scenario.researcher), + "admin": asdict(scenario.admin)} + config_path.touch(mode=0o600) + config_path.write_text(json.dumps(config)) + results_path = run_path / "browser-results.json" + results_path.unlink(missing_ok=True) + env = {**os.environ, "CODE4ME_E2E_BASE_URL": scenario.base_url, + "CODE4ME_E2E_WEB_BUILD": str(web_build), + "CODE4ME_E2E_WEB_PORT": str(port), "CODE4ME_E2E_BROWSER_CONFIG": str(config_path), + "CODE4ME_E2E_BROWSER_RESULTS": str(results_path), + "CODE4ME_E2E_BROWSER_EXPECTED_IDS": json.dumps(list(EXPECTED_SCENARIO_IDS))} + with (run_path / "browser-server.log").open("w") as log: + child = subprocess.Popen([node, str(BROWSER_DIR / "serve.js")], cwd=root, + env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if child.poll() is not None: + raise RuntimeError("Website host exited; see browser-server.log") + try: + with urllib.request.urlopen(url, timeout=2) as response: + if response.status == 200: + break + except OSError: + pass + time.sleep(0.2) + else: + raise TimeoutError("Website host did not become ready") + print(f"Testing {len(EXPECTED_SCENARIO_IDS)} website study, consent and " + "persisted-outcome checks in Chromium", file=sys.stderr) + rc = process.run([python, "-u", str(BROWSER_DIR / "scenarios.py")], cwd=root, env=env, + log_path=run_path / "browser-test.log", timeout=300) + if not results_path.is_file(): + raise RuntimeError("The website suite wrote no results; see browser-test.log") + results_text = results_path.read_text(encoding="utf-8") + assert_no_secret_material( + results_text, + secrets=[scenario.researcher.password, scenario.admin.password, + str(state.get("join_code") or "")], + ) + results = json.loads(results_text) + details["per_step"] = results.get("steps") if isinstance(results, dict) else None + steps = validate_scenario_results(results) + context = validate_context(results) + if rc: + raise RuntimeError("The website suite reported failing checks; see browser-test.log") + details["per_step"] = steps + details["context"] = context + details["persisted"] = verify_persisted_outcomes(scenario, state, context, + cookies=login_cookies(scenario)) + status = "PASS" + except PrerequisiteError as error: + status = "BLOCKED" + details["prerequisite"] = True + details["error"] = str(error) + except Exception as error: # noqa: BLE001 - a failed check is a failed layer + details["error"] = str(error) + if _workflow_touched_stack(state_path): + report.capture_backend_logs(scenario, run_path) + finally: + if child: + process.stop(child) + config_path.unlink(missing_ok=True) + if status == "BLOCKED": + fix_hint = str(details.get("error") or "missing prerequisite")[:240] + elif status == "PASS": + fix_hint = "" + else: + fix_hint = f"Inspect {run_path}/browser-test.log" + workflow._record_plugin_step(run_path, scenario, StepResult( + BROWSER_STEP_ID, status, int((time.monotonic() - started) * 1000), details, fix_hint, + )) + return 0 if status == "PASS" else 1 diff --git a/e2e/code4me_e2e/cli.py b/e2e/code4me_e2e/cli.py new file mode 100644 index 0000000..00ce417 --- /dev/null +++ b/e2e/code4me_e2e/cli.py @@ -0,0 +1,386 @@ +"""argparse CLI for the code4me-e2e harness.""" + +from __future__ import annotations + +import argparse +import json +import os +import signal +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import List, Optional + +from . import prereqs, real_agents, stack, suite, ui, workflow +from .config import AgentProbeReason, ScenarioError, load_scenario +from .paths import E2E_DIR + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="code4me_e2e", + description="Standalone end-to-end harness for the Code4Me research participant workflow.", + ) + sub = parser.add_subparsers(dest="command", required=True) + + test = sub.add_parser("test", help="run the complete automated regression gate") + test.add_argument("--layer", choices=["all", "backend", "plugin", "browser", "agents"], default="all") + test.add_argument("--scenario") + test.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") + test.add_argument("--run-dir") + test.add_argument("--json", action="store_true", dest="as_json") + test.add_argument("--keep-stack", action="store_true") + test.add_argument("--no-setup", action="store_true", dest="no_setup", + help="check prerequisites without provisioning anything (missing ones block their layers)") + + setup = sub.add_parser( + "setup", + help="check and provision every host prerequisite of a layer (idempotent)", + ) + setup.add_argument("--layer", choices=["all", "backend", "plugin", "browser", "agents"], default="all") + setup.add_argument("--check", action="store_true", help="report only; start, build or install nothing") + setup.add_argument("--scenario", default=None) + setup.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") + setup.add_argument("--json", action="store_true", dest="as_json") + + stack_parser = sub.add_parser("stack", help="manage the disposable backend stack") + stack_parser.add_argument("action", choices=["up", "down", "status"]) + stack_parser.add_argument("--project-name", default=None) + stack_parser.add_argument("--scenario", default=None) + stack_parser.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") + + doctor = sub.add_parser("doctor", help="diagnose a backend before running the workflow") + doctor.add_argument("--base-url", default=None) + doctor.add_argument("--scenario", default=None) + doctor.add_argument("--project-name", default=None) + doctor.add_argument("--json", action="store_true", dest="as_json") + doctor.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") + + run = sub.add_parser("run", help="run the end-to-end workflow") + run.add_argument("--scenario", default=None) + run.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") + run.add_argument("--only", default=None, help="comma-separated step ids") + run.add_argument("--from", dest="from_step", default=None, help="start at this step") + run.add_argument("--json", action="store_true", dest="as_json") + run.add_argument("--run-dir", default=None) + run.add_argument("--keep-stack", action="store_true") + + step = sub.add_parser("step", help="run (or re-run) one workflow step") + step.add_argument("step_id") + step.add_argument("--state", default=None, help="path to an existing state.json") + step.add_argument("--run-dir", default=None) + step.add_argument("--scenario", default=None) + step.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") + step.add_argument("--json", action="store_true", dest="as_json") + + plugin_test = sub.add_parser( + "plugin-test", + help="run the plugin's own Kotlin code in an IntelliJ fixture against the live backend", + ) + plugin_test.add_argument("--run-dir", default=None) + plugin_test.add_argument("--scenario", default=None) + plugin_test.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") + plugin_test.add_argument("--json", action="store_true", dest="as_json") + plugin_test.add_argument("--keep-stack", action="store_true") + + ui_test = sub.add_parser( + "ui-test", + help="boot a real IntelliJ sandbox and drive the plugin through robot-server", + ) + ui_test.add_argument("--run-dir", default=None) + ui_test.add_argument("--scenario", default=None) + ui_test.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") + ui_test.add_argument("--json", action="store_true", dest="as_json") + ui_test.add_argument("--keep-stack", action="store_true") + ui_test.add_argument( + "--keep-ide", + action="store_true", + help="leave the sandbox IDE running after the suite (default: stop only this harness's IDE)", + ) + + agent_probe = sub.add_parser( + "agent-probe", + help="identify and ACP-probe a host-installed Goose/Codex agent (no Compose/IDE)", + ) + agent_probe.add_argument("--framework", choices=["goose", "codex"], required=True) + agent_probe.add_argument("--scenario", default=None) + agent_probe.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") + agent_probe.add_argument("--run-dir", default=None) + agent_probe.add_argument("--executable", default=None, help="host executable override") + agent_probe.add_argument("--home", default=None, help="isolated agent home override") + agent_probe.add_argument("--timeout", type=int, default=120, help="per-request probe timeout (seconds)") + agent_probe.add_argument( + "--detect-only", + action="store_true", + dest="detect_only", + help="resolve and identify the executable; never launch a session or a model turn", + ) + agent_probe.add_argument("--json", action="store_true", dest="as_json") + agent_probe.add_argument("--local-provider", action="store_true", + help="use a local streaming model fixture and require its unique answer") + agent_probe.add_argument("--quota-exhausted", action="store_true", dest="quota_exhausted", + help="with --local-provider: the fixture answers the research gateway's " + "402 quota_exhausted; pass only on a typed quota block after one request") + + return parser + + +def run_agent_probe( + scenario, + framework: str, + *, + run_dir: Optional[str] = None, + executable: Optional[str] = None, + home: Optional[str] = None, + timeout: int = 120, + detect_only: bool = False, + as_json: bool = False, + local_provider: bool = False, + quota_exhausted: bool = False, +) -> int: + """Standalone probe for a real participant-installed agent. + + Uses no Compose stack and no IDE. ``BLOCKED`` exits nonzero: a missing + prerequisite can never be reported as a pass. + """ + if timeout < 1: + raise ScenarioError("--timeout must be >= 1") + agent = scenario.agent + agent_executable = executable or agent.executable + agent_home = home or agent.agent_home + agent_args = list(agent.agent_command_args) if agent.agent_command_args else None + explicit = agent_executable or agent.agent_command or agent.agent_package or \ + os.environ.get(real_agents.ENV_OVERRIDES.get(framework, ""), "") + blocked = None + if framework == "codex" and not explicit: + # Default to the plugin's vendored Codex ACP adapter, the adapter the + # plugin ships, never silently to whatever is on PATH. The probe runs + # outside the workspace lock, so it never builds the adapter itself; + # only its own launcher is (re)written, and not by --detect-only. + resolved: dict = {} + check = prereqs.check_codex_acp(False, executables=resolved, write_wrapper=not detect_only) + if resolved.get("codex"): + agent_executable = resolved["codex"] + else: + blocked = real_agents.ProbeResult.blocked( + framework, AgentProbeReason.MISSING_BINARY, + f"the vendored Codex ACP adapter is not ready ({check.status}: {check.detail}); run " + "`python3 -m code4me_e2e setup --layer agents`, or pass --executable / set " + "CODE4ME_E2E_CODEX_EXECUTABLE for another codex-acp", + protocol={"adapter_check": check.to_dict()}, + ) + + if blocked is not None: + result = blocked + elif detect_only: + result = real_agents.probe_identity( + framework, + executable=agent_executable, + command=agent.agent_command, + package=agent.agent_package, + ) + else: + from . import acp_probe + + run_path = Path(run_dir).resolve() if run_dir else ( + E2E_DIR / ".cache/agent-probe" / + f"{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}-{framework}" + ) + if local_provider: + from . import native_agents + result = native_agents.probe(framework, run_path, executable=agent_executable, timeout=timeout, + command=agent.agent_command, package=agent.agent_package, + argv=agent_args, home=agent_home, quota_exhausted=quota_exhausted) + else: + result = acp_probe.run_probe( + framework, + run_dir=run_path, + executable=agent_executable, + command=agent.agent_command, + package=agent.agent_package, + argv=agent_args, + home=agent_home, + prompt=scenario.message.prompt, + timeout=timeout, + ) + + if as_json: + print(json.dumps(result.to_dict(), indent=2, default=str)) + else: + print(f"{result.framework} {result.status}" + (f" ({result.reason})" if result.reason else "")) + if result.identity is not None: + print(f" executable: {result.identity.path} source={result.identity.source}") + print(f" sha256: {result.identity.sha256} size={result.identity.size} version={result.identity.version}") + if result.detail: + print(f" detail: {result.detail}") + for check in result.checks: + print(f" check: {json.dumps(dict(check), sort_keys=True)}") + return 0 if result.passed else 1 + + +def _load(args: argparse.Namespace): + overrides = list(getattr(args, "set", []) or []) + base_url = getattr(args, "base_url", None) + if base_url: + overrides = [f"base_url={base_url}"] + overrides + scenario = load_scenario(args.scenario, overrides) + project = getattr(args, "project_name", None) + if project: + from .config import finalize_scenario + scenario.stack.project_name = project + finalize_scenario(scenario) + return scenario + + +def _main(argv: Optional[List[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + scenario = _load(args) + except ScenarioError as error: + print(f"scenario error: {error}", file=sys.stderr) + return 2 + + if args.command not in ("doctor", "agent-probe") and not workflow._own_stack(scenario): + raise ScenarioError("Mutating tests require the disposable stack's loopback base_url; doctor can inspect other backends") + + if args.command == "test": + return suite.run(scenario, run_dir=args.run_dir, keep_stack=args.keep_stack, + as_json=args.as_json, layer=args.layer, provision=not args.no_setup) + + if args.command == "setup": + planned = suite.plan_layers(args.layer) + result = prereqs.prepare(scenario, planned, provision=not args.check) + if args.as_json: + print(json.dumps(result.to_dict(), indent=2)) + else: + prereqs.print_table(result, stream=sys.stdout) + for layer_name, state in result.to_dict()["layers"].items(): + print(f"layer {layer_name:<8} {state}") + return 0 if result.ready else 1 + + if args.command == "stack": + if args.action == "up": + print(stack.up(scenario)) + return 0 + if args.action == "down": + print(stack.down(scenario)) + return 0 + print(stack.ps(scenario)) + return 0 + + if args.command == "doctor": + return workflow.run_doctor(scenario, as_json=args.as_json) + + if args.command == "agent-probe": + return run_agent_probe( + scenario, + args.framework, + run_dir=args.run_dir, + executable=args.executable, + home=args.home, + timeout=args.timeout, + detect_only=args.detect_only, + as_json=args.as_json, + local_provider=args.local_provider, + quota_exhausted=args.quota_exhausted, + ) + + if args.command == "run": + if args.run_dir and str(args.run_dir).endswith("state.json"): + args.run_dir = str(Path(args.run_dir).parent) + return workflow.run_workflow( + scenario, + only=args.only, + from_step=args.from_step, + run_dir=args.run_dir, + keep_stack=args.keep_stack, + as_json=args.as_json, + ) + + if args.command == "step": + run_dir = args.run_dir + if args.state: + run_dir = str(Path(args.state).parent) + return workflow.run_single_step( + scenario, args.step_id, run_dir=run_dir, as_json=args.as_json + ) + + if args.command == "plugin-test": + run_dir = args.run_dir + if run_dir and str(run_dir).endswith("state.json"): + run_dir = str(Path(run_dir).parent) + return workflow.run_plugin_test( + scenario, + run_dir=run_dir, + keep_stack=args.keep_stack, + as_json=args.as_json, + ) + + if args.command == "ui-test": + run_dir = args.run_dir + if run_dir and str(run_dir).endswith("state.json"): + run_dir = str(Path(run_dir).parent) + return ui.run_ui_test( + scenario, + run_dir=run_dir, + keep_ide=args.keep_ide, + keep_stack=args.keep_stack, + as_json=args.as_json, + ) + + parser.error("unknown command") + return 2 + + +class _Terminated(KeyboardInterrupt): + """SIGTERM, unwound like Ctrl-C so every ``finally`` stops what this run started.""" + + +def _raise_terminated(signum, _frame): + raise _Terminated(f"terminated by signal {signum}") + + +def main(argv: Optional[List[str]] = None) -> int: + # A `kill` or an agent's tool timeout sends SIGTERM. Without a handler the + # interpreter dies at once and detached builds, installs or the sandbox IDE + # keep running; unwinding runs the cleanup that Ctrl-C already runs. + try: + previous = signal.signal(signal.SIGTERM, _raise_terminated) + except ValueError: # not the main thread: keep the default behaviour + previous = None + try: + return _locked_main(argv) + finally: + if previous is not None: + signal.signal(signal.SIGTERM, previous) + + +def _locked_main(argv: Optional[List[str]]) -> int: + import fcntl + # The Gradle sandbox and Compose ports are shared across invocations. The + # agent probe uses neither, so it must run even while a gate is active. + tokens = list(argv) if argv is not None else sys.argv[1:] + if tokens[:1] == ["agent-probe"]: + return _execute(argv) + cache = stack.E2E_DIR / ".cache" + cache.mkdir(exist_ok=True) + with (cache / "run.lock").open("w") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print("Another e2e command is running in this workspace.", file=sys.stderr) + return 2 + return _execute(argv) + + +def _execute(argv: Optional[List[str]]) -> int: + try: + return _main(argv) + except (ScenarioError, workflow.WorkflowError, stack.StackError, OSError) as error: + print(f"e2e error: {error}", file=sys.stderr) + return 2 + except _Terminated: + return 143 + except KeyboardInterrupt: + return 130 diff --git a/e2e/code4me_e2e/config.py b/e2e/code4me_e2e/config.py new file mode 100644 index 0000000..6de9966 --- /dev/null +++ b/e2e/code4me_e2e/config.py @@ -0,0 +1,383 @@ +"""Scenario configuration for the code4me-e2e harness. + +Every value the harness uses is declared here, is overridable from a JSON +scenario file and from ``--set dotted.key=value``, and is documented in +``scenario.example.json`` and ``README.md``. + +The defaults are chosen so ``python3 -m code4me_e2e run`` works with no arguments +against the harness's own disposable stack. +""" + +from __future__ import annotations + +import hashlib +import ipaddress +import json +import platform as host_platform +import re +from dataclasses import asdict, dataclass, field, is_dataclass +from enum import Enum +from typing import Any, List, Optional + +#: Sentinel secret value stored in run state / reports as ````. +REDACTED = "" + + +class AgentProbeReason(str, Enum): + """Typed reasons a real-agent (Goose/Codex) probe can be BLOCKED. + + A missing prerequisite must surface one of these instead of a pass or a + bare crash. + """ + + MISSING_BINARY = "missing_binary" + MISSING_AUTH = "missing_auth" + QUOTA = "quota" + PROTOCOL = "protocol" + + +def sha256_of(*parts: str) -> str: + """Return ``sha256:<64 lowercase hex>`` over the UTF-8 parts.""" + digest = hashlib.sha256() + for part in parts: + digest.update(str(part).encode("utf-8")) + return "sha256:" + digest.hexdigest() + + +@dataclass +class StackConfig: + """The disposable docker-compose stack the harness provisions.""" + + project_name: str = "code4me-e2e" + backend_port: int = 28008 + db_port: int = 25432 + redis_port: int = 26379 + image: str = "code4me2-server-backend:latest" + bootstrap_signing_secret: str = "e2e-bootstrap-secret" + stub_secret_env: str = "E2E_STUB_API_KEY" + stub_secret_value: str = "e2e-stub-secret-value" + db_password: str = "postgres" + db_name: str = "code4me_e2e" + db_user: str = "postgres" + #: Fixed host port for the in-process stub provider. A fixed port keeps + #: ``--from``/``step`` resumption working across processes. + stub_port: int = 28999 + #: Interface the stub provider listens on. The backend container calls it + #: through ``host.docker.internal``: Docker Desktop and Colima forward that + #: to the host's loopback, but a Linux Docker engine maps it to the bridge + #: gateway, which reaches the stub only when it listens on ``0.0.0.0``. + stub_host: str = "127.0.0.1" + + +@dataclass +class AccountConfig: + email: str + password: str + name: str + + +@dataclass +class StudyConfig: + name: str = "E2E Synthetic Study" + description: str = "Synthetic study created by the code4me-e2e harness." + owner: str = "e2e-research-ops" + start_at: Optional[str] = None + end_at: Optional[str] = None + + +@dataclass +class AgentConfig: + model: str = "e2e-stub-model" + framework_version: str = "code4me2-agent" + #: Supported values (see agent/profiles.py): auto | per_step | suggestion_only + approval_policy: str = "auto" + max_steps: int = 4 + agent_id: str = "code4me2-e2e-agent" + release_id: str = "e2e-release-1" + release_version: str = "1.0.0" + artifact_digest: Optional[str] = None + artifact_path: Optional[str] = None + artifact_size: int = 1024 + #: Inputs of the standalone ``agent-probe`` (``--layer agents`` probes both + #: frameworks and only honours ``CODE4ME_E2E__EXECUTABLE``). + #: ``agent_command``/``agent_package`` name a participant-installed agent the + #: way a BYOA release declares it; ``agent_command_args`` replaces the + #: framework's default ACP argv. ``executable`` replaces discovery + #: (``CODE4ME_E2E__EXECUTABLE`` also works) and ``agent_home`` + #: replaces the per-run isolated agent home. + agent_command: Optional[str] = None + agent_command_args: List[str] = field(default_factory=list) + agent_package: Optional[str] = None + executable: Optional[str] = None + agent_home: Optional[str] = None + #: Name of the agent profile created by the ``create_profile`` step. A + #: distinct name lets one stack carry several self-consistent e2e identities + #: (for example the ``plugin-test`` release and the ``ui-test`` release) + #: without an existing profile pinning the wrong release artifact digest. + profile_name: str = "e2e-profile" + #: Tools the managed run policy allows. The managed runtime advertises its + #: full tool set on every turn, so a profile with no tools makes the relay + #: reject the turn with a 403 tool-policy refusal (harness scenario gap). + tools: list = field(default_factory=lambda: [ + "read_file", "write_file", "create_file", "replace_text", + "list_files", "search_files", "run_command", + ]) + adapter_id: Optional[str] = None + adapter_version: str = "1.0.0" + adapter_digest: Optional[str] = None + + +@dataclass +class PlatformConfig: + os: str = field(default_factory=lambda: {"Darwin": "macos", "Windows": "windows"}.get(host_platform.system(), "linux")) + arch: str = field(default_factory=lambda: "aarch64" if host_platform.machine().lower() in ("arm64", "aarch64") else "x64") + ide_build: str = "IU-243.1" + host_kind: str = "IntelliJ IDEA" + plugin_version: str = "e2e" + + +@dataclass +class MessageConfig: + prompt: str = "Say hello in one short sentence." + #: When null the stub provider's own deterministic token is asserted. + expected_substring: Optional[str] = None + + +@dataclass +class TimeoutConfig: + backend_ready_seconds: int = 300 + step_seconds: int = 120 + + +@dataclass +class Scenario: + #: Backend origin. Defaults to http://localhost:. + base_url: Optional[str] = None + stack: StackConfig = field(default_factory=StackConfig) + admin: AccountConfig = field( + default_factory=lambda: AccountConfig( + "e2e-admin@example.com", "AdminPass123", "E2E Admin" + ) + ) + researcher: AccountConfig = field( + default_factory=lambda: AccountConfig( + "e2e-researcher@example.com", "ResearcherPass123", "E2E Researcher" + ) + ) + participant: AccountConfig = field( + default_factory=lambda: AccountConfig( + "e2e-participant@example.com", "ParticipantPass123", "E2E Participant" + ) + ) + study: StudyConfig = field(default_factory=StudyConfig) + agent: AgentConfig = field(default_factory=AgentConfig) + platform: PlatformConfig = field(default_factory=PlatformConfig) + message: MessageConfig = field(default_factory=MessageConfig) + timeouts: TimeoutConfig = field(default_factory=TimeoutConfig) + + +class ScenarioError(ValueError): + """Raised for an unusable scenario document or override. + + ``code`` is a stable machine-readable reason (empty for plain messages). + """ + + def __init__(self, message: str, *, code: str = ""): + super().__init__(message) + self.code = code + + +# --------------------------------------------------------------------------- +# Loading and overrides +# --------------------------------------------------------------------------- + + +def _merge_into(target: Any, data: Any, path: str) -> None: + """Recursively merge a JSON mapping onto a dataclass instance.""" + if not isinstance(data, dict): + raise ScenarioError(f"{path or 'scenario'} must be a JSON object") + for key, value in data.items(): + if key.startswith("$") or key.startswith("_"): + continue # documentation-only keys (comments) in the example scenario + if not hasattr(target, key): + raise ScenarioError(f"unknown scenario key {path + '.' if path else ''}{key}") + current = getattr(target, key) + if is_dataclass(current) and not isinstance(current, type): + _merge_into(current, value, f"{path + '.' if path else ''}{key}") + else: + setattr(target, key, value) + + +def _coerce_override(value: str) -> Any: + """Parse an override value as JSON when possible, else keep it a string.""" + try: + return json.loads(value) + except (TypeError, ValueError): + return value + + +def apply_overrides(scenario: Scenario, overrides: List[str]) -> None: + """Apply ``dotted.key=value`` overrides in place.""" + for item in overrides: + if "=" not in item: + raise ScenarioError(f"--set expects dotted.key=value, got {item!r}") + dotted, _, raw = item.partition("=") + parts = dotted.strip().split(".") + target: Any = scenario + for part in parts[:-1]: + if not hasattr(target, part): + raise ScenarioError(f"unknown scenario key {dotted!r}") + target = getattr(target, part) + leaf = parts[-1] + if not hasattr(target, leaf): + raise ScenarioError(f"unknown scenario key {dotted!r}") + setattr(target, leaf, _coerce_override(raw)) + + +def load_scenario(path: Optional[str] = None, overrides: Optional[List[str]] = None) -> Scenario: + """Build a scenario from defaults, an optional JSON file and overrides.""" + scenario = Scenario() + if path: + try: + with open(path, "r", encoding="utf-8") as handle: + # Strip keys the example uses for documentation only. + raw = json.load(handle) + except OSError as error: + raise ScenarioError(f"cannot read scenario file {path!r}: {error}") from error + except json.JSONDecodeError as error: + raise ScenarioError(f"scenario file {path!r} is not valid JSON: {error}") from error + if not isinstance(raw, dict): + raise ScenarioError("scenario must be a JSON object") + raw.pop("$schema", None) + _merge_into(scenario, raw, "") + apply_overrides(scenario, overrides or []) + finalize_scenario(scenario) + return scenario + + +def finalize_scenario(scenario: Scenario) -> Scenario: + """Fill derived defaults and validate the document.""" + if not scenario.base_url: + scenario.base_url = f"http://localhost:{scenario.stack.backend_port}" + + agent = scenario.agent + platform = scenario.platform + if not agent.artifact_digest: + agent.artifact_digest = sha256_of( + "code4me2://e2e/artifact", agent.agent_id, agent.release_id, platform.os, platform.arch + ) + if not agent.artifact_path: + agent.artifact_path = f"agents/{platform.os}-{platform.arch}/code4me2-agent" + if not agent.adapter_id: + agent.adapter_id = f"{agent.agent_id}-adapter" + if not agent.adapter_digest: + agent.adapter_digest = sha256_of( + "code4me2://e2e/adapter", agent.agent_id, agent.release_version + ) + + _validate(scenario) + return scenario + + +def _validate(scenario: Scenario) -> None: + if not re.fullmatch(r"code4me-e2e(?:-[a-z0-9-]+)?", scenario.stack.project_name): + raise ScenarioError("stack.project_name must be code4me-e2e or code4me-e2e- (reserved disposable projects)") + ports = [getattr(scenario.stack, name) for name in ("backend_port", "db_port", "redis_port", "stub_port")] + if any(type(port) is not int or not 1 <= port <= 65535 for port in ports) or len(set(ports)) != len(ports): + raise ScenarioError("stack ports must be distinct integers from 1 to 65535") + try: + ipaddress.IPv4Address(scenario.stack.stub_host if type(scenario.stack.stub_host) is str else "") + except ValueError: + raise ScenarioError("stack.stub_host must be an IPv4 address such as 127.0.0.1 or 0.0.0.0") from None + for role in ("admin", "researcher", "participant"): + account: AccountConfig = getattr(scenario, role) + if not account.email or "@" not in account.email: + raise ScenarioError(f"{role}.email must be an email address") + # Mirror Queries.CreateUser's validator so `create_accounts` cannot be + # rejected by a surprising 422. + if not re.match(r"^(?=.*[A-Z])(?=.*[a-z])(?=.*\d)\S{8,50}$", account.password): + raise ScenarioError( + f"{role}.password must be 8-50 chars with upper, lower and a digit" + ) + if not (3 <= len(account.name) <= 50): + raise ScenarioError(f"{role}.name must be 3-50 characters") + if scenario.agent.approval_policy not in ("auto", "per_step", "suggestion_only"): + raise ScenarioError( + "agent.approval_policy must be one of auto, per_step, suggestion_only" + ) + if scenario.agent.framework_version not in ("code4me2-agent", "goose", "codex"): + raise ScenarioError("agent.framework_version must be code4me2-agent, goose or codex") + if scenario.agent.framework_version != "code4me2-agent": + # Workflow releases come from the packaged producer manifest; Goose/Codex + # arms are BYOA and not managed by the relay, so this would only fail + # later at create_profile/send_message with a server 4xx. + raise ScenarioError( + "the workflow layers import the packaged code4me2-agent release; Goose and Codex " + "are exercised by `--layer agents` and `agent-probe --framework goose|codex`", + code="FRAMEWORK_NOT_IN_WORKFLOW", + ) + if scenario.agent.max_steps < 1: + raise ScenarioError("agent.max_steps must be >= 1") + args = scenario.agent.agent_command_args + if not isinstance(args, list) or any(not isinstance(item, str) or not item or "\x00" in item for item in args): + raise ScenarioError("agent.agent_command_args must be a list of non-empty strings without NUL", + code="AGENT_COMMAND_ARGS_INVALID") + for key in ("executable", "agent_home", "agent_command", "agent_package"): + value = getattr(scenario.agent, key) + if value is not None and (not isinstance(value, str) or not value.strip()): + raise ScenarioError(f"agent.{key} must be a non-empty string or null") + + +def scenario_dict(scenario: Scenario) -> dict: + """Return the scenario as a plain dict (secrets intact).""" + return asdict(scenario) + + +#: Case-insensitive key fragments redacted from reports/logs. +_SECRET_KEYS = ( + "password", + "secret", + "token", + "grant", + "capability", + "signature", +) +_SAFE_KEYS = { + "stubsecretenv", + "maxtokens", + "maxcontexttokens", + "usagetokens", + "prompttokens", + "completiontokens", + "totaltokens", + "tokenhookactivationinseconds", + "authenticationtokenexpiresinseconds", + "sessiontokenexpiresinseconds", + "emailverificationtokenexpiresinseconds", + "resetpasswordtokenexpiresinseconds", + "tokenexpiresinseconds", +} + + +def is_secret_key(key: str) -> bool: + normalized = "".join(ch for ch in str(key).lower() if ch.isalnum()) + if normalized in _SAFE_KEYS: + return False + return any(fragment in normalized for fragment in _SECRET_KEYS) + + +def redact(value: Any) -> Any: + """Recursively replace secret-shaped values with ````.""" + if isinstance(value, dict): + return { + key: (REDACTED if is_secret_key(key) else redact(item)) + for key, item in value.items() + } + if isinstance(value, list): + return [redact(item) for item in value] + return value + + +def sanitized_scenario(scenario: Scenario) -> dict: + """A report-safe scenario: passwords/secrets replaced with ````.""" + return redact(scenario_dict(scenario)) diff --git a/e2e/code4me_e2e/http.py b/e2e/code4me_e2e/http.py new file mode 100644 index 0000000..ed09d0d --- /dev/null +++ b/e2e/code4me_e2e/http.py @@ -0,0 +1,242 @@ +"""Minimal stdlib HTTP client with a cookie jar and secret redaction. + +No third-party dependency: ``urllib.request`` plus a hand-rolled cookie jar for +the three cookies the Code4Me backend uses (``auth_token``, ``session_token``, +``project_token``). HTTP errors never raise: every call returns +``(status, parsed_json, raw_text)`` so a step can assert on a status code. + +Every captured exchange is sanitized before it is stored, so the run report and +logs never contain a password, session cookie, launch grant or capability. +""" + +from __future__ import annotations + +import http.cookies +import json +import time +import urllib.error +import urllib.request +import uuid +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Tuple + +from .config import is_secret_key, redact + +#: Best-effort redaction of raw (non-JSON) bodies. +_REDACT_PATTERNS = ( + (r"(?i)(Bearer)\s+[A-Za-z0-9._\-]+", r"\1 "), + (r"(?i)(auth_token|session_token|project_token|acp_token|grant)=([^;\s]+)", r"\1="), +) + + +class HttpError(RuntimeError): + """The HTTP call could not be completed (DNS, refused, timeout).""" + + +@dataclass +class HttpResponse: + status: int + json: Optional[Any] + text: str + headers: Dict[str, str] = field(default_factory=dict) + + def body_contains(self, needle: str) -> bool: + return needle in self.text + + +def redact_text(text: str) -> str: + """Best-effort redaction for a raw body that is not a JSON object.""" + import re + + result = text + for pattern, replacement in _REDACT_PATTERNS: + result = re.sub(pattern, replacement, result) + return result + + +def encode_multipart( + fields: Dict[str, str], files: List[Tuple[str, str, bytes]] +) -> Tuple[bytes, str]: + """Encode one multipart/form-data body using only the standard library. + + ``files`` entries are ``(field_name, filename, payload)``. The release import + endpoint takes the manifest as a text field and each archive as a file field, + so the harness must speak multipart without adding a dependency. + """ + boundary = "----code4me-e2e-" + uuid.uuid4().hex + chunks: List[bytes] = [] + for name, value in fields.items(): + chunks.append( + ( + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="{name}"\r\n\r\n' + f"{value}\r\n" + ).encode("utf-8") + ) + for name, filename, payload in files: + chunks.append( + ( + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="{name}"; filename="{filename}"\r\n' + "Content-Type: application/octet-stream\r\n\r\n" + ).encode("utf-8") + ) + chunks.append(payload) + chunks.append(b"\r\n") + chunks.append(f"--{boundary}--\r\n".encode("utf-8")) + return b"".join(chunks), f"multipart/form-data; boundary={boundary}" + + +class HttpClient: + """A cookie-aware HTTP client for one authenticated role.""" + + def __init__( + self, + base_url: str, + *, + label: str = "client", + timeout: float = 30.0, + on_record: Optional[Callable[[Dict[str, Any]], None]] = None, + ) -> None: + self.base_url = base_url.rstrip("/") + self.label = label + self.timeout = timeout + self.cookies: Dict[str, str] = {} + self.records: list[Dict[str, Any]] = [] + self._on_record = on_record + + # -- cookie jar -------------------------------------------------------- + + def set_cookies(self, cookies: Dict[str, str]) -> None: + self.cookies.update({k: v for k, v in cookies.items() if v}) + + def export_cookies(self) -> Dict[str, str]: + return dict(self.cookies) + + def _absorb_cookies(self, headers) -> None: + values = headers.get_all("Set-Cookie") or [] + for value in values: + jar = http.cookies.SimpleCookie() + try: + jar.load(value) + except http.cookies.CookieError: + continue + for name, morsel in jar.items(): + self.cookies[name] = morsel.value + + # -- requests ---------------------------------------------------------- + + def request( + self, + method: str, + path: str, + *, + json_body: Any = None, + body: Optional[bytes] = None, + content_type: Optional[str] = None, + record_request: Any = None, + headers: Optional[Dict[str, str]] = None, + bearer: Optional[str] = None, + timeout: Optional[float] = None, + ) -> HttpResponse: + url = self.base_url + (path if path.startswith("/") else "/" + path) + request_headers = {"Accept": "application/json"} + data: Optional[bytes] = None + if body is not None: + data = body + if content_type: + request_headers["Content-Type"] = content_type + elif json_body is not None: + data = json.dumps(json_body).encode("utf-8") + request_headers["Content-Type"] = "application/json" + if bearer: + request_headers["Authorization"] = f"Bearer {bearer}" + if self.cookies: + request_headers["Cookie"] = "; ".join( + f"{key}={value}" for key, value in self.cookies.items() + ) + if headers: + request_headers.update(headers) + + request = urllib.request.Request( + url, data=data, headers=request_headers, method=method + ) + started = time.monotonic() + try: + with urllib.request.urlopen( + request, timeout=timeout or self.timeout + ) as response: + status = response.status + raw = response.read().decode("utf-8", "replace") + response_headers = response.headers + except urllib.error.HTTPError as error: + status = error.code + raw = error.read().decode("utf-8", "replace") + response_headers = error.headers + except (urllib.error.URLError, TimeoutError, OSError) as error: + raise HttpError(f"{method} {path} failed: {error}") from error + duration_ms = int((time.monotonic() - started) * 1000) + + self._absorb_cookies(response_headers) + + parsed: Optional[Any] = None + if raw.strip(): + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = None + + record = { + "client": self.label, + "method": method, + "path": path, + "status": status, + "duration_ms": duration_ms, + "request": ( + redact(json_body) + if json_body is not None + else (redact(record_request) if record_request is not None else None) + ), + "response": ( + redact(parsed) if parsed is not None else redact_text(raw)[:4000] + ), + } + if self._on_record is not None: + self._on_record(record) + else: + self.records.append(record) + + return HttpResponse(status, parsed, raw, dict(response_headers.items())) + + # Convenience wrappers ------------------------------------------------- + + def get(self, path: str, **kwargs: Any) -> HttpResponse: + return self.request("GET", path, **kwargs) + + def post(self, path: str, json_body: Any = None, **kwargs: Any) -> HttpResponse: + return self.request("POST", path, json_body=json_body, **kwargs) + + def put(self, path: str, json_body: Any = None, **kwargs: Any) -> HttpResponse: + return self.request("PUT", path, json_body=json_body, **kwargs) + + def post_multipart( + self, + path: str, + *, + fields: Dict[str, str], + files: List[Tuple[str, str, bytes]], + **kwargs: Any, + ) -> HttpResponse: + """POST multipart/form-data; the record keeps only field/file names.""" + body, content_type = encode_multipart(fields, files) + return self.request( + "POST", + path, + body=body, + content_type=content_type, + record_request={"fields": sorted(fields), "files": [name for _, name, _ in files]}, + **kwargs, + ) + + def head(self, path: str, **kwargs: Any) -> HttpResponse: + return self.request("HEAD", path, **kwargs) diff --git a/e2e/code4me_e2e/native_agents.py b/e2e/code4me_e2e/native_agents.py new file mode 100644 index 0000000..8b8e316 --- /dev/null +++ b/e2e/code4me_e2e/native_agents.py @@ -0,0 +1,158 @@ +"""Real installed agents against a local provider; separate from the IDE layer.""" +from __future__ import annotations + +import secrets +import time +from typing import TYPE_CHECKING, Dict, Optional + +from . import acp_probe, real_agents, workflow +from .agent_provider import GATEWAY_CHAT_ROUTE, AgentProvider +from .config import AgentProbeReason +from .steps import StepResult + +if TYPE_CHECKING: + from pathlib import Path + +#: Goose 1.51 makes at most two provider calls per prompt turn: the reply and +#: the session-description (title) call. More gateway requests than that in +#: quota mode means the agent retried a refusal it must not retry. +GOOSE_CALLS_PER_TURN = 2 + +#: A deliberately wrong Goose config: another provider and a dead host. The +#: study launch must override it through the environment (the plugin's runtime +#: bindings), or the probe fails; this is the bypass hardening the research +#: proxy relies on. +POISONED_GOOSE_CONFIG = ( + "GOOSE_PROVIDER: anthropic\n" + "GOOSE_MODEL: poisoned-model\n" + "OPENAI_HOST: http://127.0.0.1:9\n" + "ANTHROPIC_HOST: http://127.0.0.1:9\n" +) + + +def _poison_goose_home(home: "Path") -> None: + config_dir = home / ".config" / "goose" + config_dir.mkdir(parents=True, exist_ok=True) + (config_dir / "config.yaml").write_text(POISONED_GOOSE_CONFIG, encoding="utf-8") + + +def probe(framework, run_path: Path, *, executable=None, timeout=120, + command=None, package=None, argv=None, home=None, quota_exhausted=False): + """Require a fresh provider receipt AND its unique answer over real ACP. + + Goose runs in the *gateway shape* a study arm receives: ``OPENAI_HOST`` is + the provider origin, ``OPENAI_BASE_PATH`` the research gateway path, the + API key a random bearer the provider must see back, ``GOOSE_PROVIDER`` + selected through the environment and ``GOOSE_PATH_ROOT`` isolating Goose's + own state, while the isolated home carries a poisoned ``config.yaml``. + + With ``quota_exhausted`` the provider answers the research gateway's + ``402 quota_exhausted`` refusal; the probe then passes only when the agent + reports a typed ``quota`` block after no more than its normal calls for + one turn (``GOOSE_CALLS_PER_TURN``; no retry storm), which is how a used-up + participant budget must look. + """ + bearer = "c4m-" + secrets.token_urlsafe(24) if framework == "goose" else None + provider = AgentProvider(expected_bearer=bearer, quota_exhausted=quota_exhausted) + provider.start() + try: + origin = provider.base_url + extra = {"OPENAI_API_KEY": bearer or "e2e-placeholder", "NO_PROXY": "127.0.0.1,localhost"} + prepare_home = None + if framework == "goose": + extra.update(GOOSE_PROVIDER="openai", GOOSE_MODEL="e2e-stub-model", + OPENAI_HOST=origin, OPENAI_BASE_PATH=GATEWAY_CHAT_ROUTE.lstrip("/"), + GOOSE_DISABLE_KEYRING="1") + # The plugin's state_dir binding: Goose's own config/data/state live + # in a directory of ours, not in the (poisoned) participant home. + extra["GOOSE_PATH_ROOT"] = str(run_path / "goose-root") + route = GATEWAY_CHAT_ROUTE + prepare_home = _poison_goose_home + elif framework == "codex": + extra.update(CODEX_PROXY_URL=origin + "/v1", CODEX_MODEL="e2e-stub-model", + OPENAI_BASE_URL=origin + "/v1") + route = "/v1/responses" + else: + raise ValueError(f"unsupported framework: {framework}") + before = provider.request_count() + result = acp_probe.run_probe( + framework, run_dir=run_path, executable=executable, timeout=timeout, + command=command, package=package, argv=argv, home=home, + prompt="Say hello in one short sentence. Do not use tools.", + env_extra=extra, expected_substring=provider.token, + prepare_home=prepare_home, + ) + requests = provider.requests_since(before) + gateway_requests = [item for item in requests if item.get("route") == route] + protocol_extra = { + "provider": "local deterministic fixture", + "provider_requests": requests, + "expected_answer_received": result.passed, + "gateway_shape": framework == "goose", + "quota_exhausted_mode": bool(quota_exhausted), + "coverage": "installed agent ACP transport; IDE and backend covered separately", + } + if quota_exhausted: + # A pass here is the typed quota block after exactly one request. + blocked_on_quota = result.blocked and result.reason == AgentProbeReason.QUOTA.value + if blocked_on_quota and 1 <= len(gateway_requests) <= GOOSE_CALLS_PER_TURN: + return real_agents.ProbeResult( + **{**result.__dict__, "status": "PASS", "reason": None, + "detail": "the agent surfaced the gateway's 402 quota refusal without retrying", + "protocol": {**result.protocol, **protocol_extra}}, + ) + return real_agents.ProbeResult.blocked( + framework, AgentProbeReason.PROTOCOL, + ( + f"expected a typed quota block after at most {GOOSE_CALLS_PER_TURN} gateway " + f"requests (one turn, no retries), got status={result.status} " + f"reason={result.reason} requests={len(gateway_requests)}" + ), + identity=result.identity, checks=result.checks, + protocol={**result.protocol, **protocol_extra}, + ) + if result.passed and not gateway_requests: + return real_agents.ProbeResult.blocked( + framework, AgentProbeReason.PROTOCOL, + "No request reached this probe's local provider", identity=result.identity, + checks=result.checks, + ) + if result.passed and framework == "goose" and not all( + item.get("auth_matched") is True for item in gateway_requests + ): + return real_agents.ProbeResult.blocked( + framework, AgentProbeReason.PROTOCOL, + "The agent did not present the gateway bearer it was given", + identity=result.identity, checks=result.checks, + ) + return real_agents.ProbeResult( + **{**result.__dict__, "protocol": {**result.protocol, **protocol_extra}}, + ) + finally: + provider.stop() + + +def run(scenario, run_path: Path, *, executables: Optional[Dict[str, str]] = None) -> int: + """Attempt both independent agents; neither missing nor blocked can pass. + + ``executables`` carries what the prerequisite step resolved (the vendored + Codex ACP adapter); anything absent is discovered on the host. + """ + started = time.monotonic() + results = [] + for framework in ("goose", "codex"): + try: + result = probe(framework, run_path / "native-agents" / framework, + executable=(executables or {}).get(framework), + timeout=scenario.timeouts.step_seconds) + results.append(result.to_dict()) + except Exception as error: + results.append({"framework": framework, "status": "FAIL", + "detail": real_agents.redact_failure_text(str(error))}) + passed = all(item["status"] == "PASS" for item in results) + status = "PASS" if passed else "FAIL" if any(item["status"] == "FAIL" for item in results) else "BLOCKED" + workflow._record_plugin_step(run_path, scenario, StepResult( + "agents_test", status, int((time.monotonic() - started) * 1000), + {"agents": results}, "" if passed else "Inspect agents_test results for the failed setup stage", + )) + return 0 if passed else 1 diff --git a/e2e/code4me_e2e/paths.py b/e2e/code4me_e2e/paths.py new file mode 100644 index 0000000..57a3b9a --- /dev/null +++ b/e2e/code4me_e2e/paths.py @@ -0,0 +1,66 @@ +"""Single source of truth for workspace and repository locations. + +The harness is version-controlled inside the server repository +(``code4me2-server/e2e``) but always needs the two application checkouts as +siblings of a workspace root:: + + / + code4me2/ # IntelliJ plugin + code4me2-server/ # backend + this harness + +``CODE4ME_E2E_WORKSPACE`` overrides the lookup, which is what CI uses after it +checks both repositories out side by side. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +#: Directory holding the harness itself (compose file, cache, run artifacts). +E2E_DIR: Path = Path(__file__).resolve().parent.parent + + +def _find_workspace_root() -> Path | None: + override = os.environ.get("CODE4ME_E2E_WORKSPACE") + if override: + root = Path(override).expanduser().resolve() + if not (root / "code4me2").is_dir() or not (root / "code4me2-server").is_dir(): + raise RuntimeError( + f"CODE4ME_E2E_WORKSPACE={root} must contain code4me2/ and code4me2-server/" + ) + return root + for candidate in (E2E_DIR.parent, *E2E_DIR.parents): + if (candidate / "code4me2").is_dir() and (candidate / "code4me2-server").is_dir(): + return candidate + # Not fatal at import: the harness unit tests need no application checkout, + # so resolution is deferred to require_workspace() for the layers that do. + return None + + +WORKSPACE_ROOT: Path | None = _find_workspace_root() + + +def require_workspace() -> Path: + """The workspace root, or a clear error when the checkouts are missing.""" + if WORKSPACE_ROOT is None: + raise RuntimeError( + "Could not locate the workspace root (a directory containing code4me2/ and " + "code4me2-server/). Set CODE4ME_E2E_WORKSPACE to override the lookup." + ) + return WORKSPACE_ROOT + + +def _browser_dir() -> Path: + """The browser scripts: vendored next to the harness, else the legacy root.""" + vendored = E2E_DIR / "browser" + if (vendored / "scenarios.py").is_file(): + return vendored + # Legacy developer checkout: /task07-browser next to /e2e. + legacy = E2E_DIR.parent / "task07-browser" + if (legacy / "scenarios.py").is_file(): + return legacy + return vendored + + +BROWSER_DIR: Path = _browser_dir() diff --git a/e2e/code4me_e2e/prereqs.py b/e2e/code4me_e2e/prereqs.py new file mode 100644 index 0000000..3a362db --- /dev/null +++ b/e2e/code4me_e2e/prereqs.py @@ -0,0 +1,799 @@ +"""Host prerequisites per layer: check them, provision what the harness owns. + +Nobody (human or coding agent) should have to prepare the host by hand before +the gate. :func:`prepare` checks every prerequisite of the planned layers and, +unless ``provision=False``, provides what the harness can own itself: + +* starts Colima when the Docker daemon is down and the active docker context + is a Colima one (never creates a VM or switches contexts); +* builds the CPU backend image when it is missing or its ``requirements.txt`` + differs from the checkout (content check, not a timestamp guess); +* creates ``e2e/.venv`` for the native builds (pinned like the release CI) + when no existing interpreter can import the build inputs; +* creates ``e2e/.browser-venv`` with Playwright and its Chromium (Playwright + keeps browsers in its shared per-user cache); +* installs the website's npm dependencies when ``node_modules`` is absent; +* builds the plugin's vendored Codex ACP adapter (gitignored ``node_modules`` / + ``dist``) and exposes it through a harness-owned wrapper; +* stops sandbox IDEs left behind by an earlier, interrupted harness run. + +One-time host tools (Docker or Colima, a JDK, Node.js, Python, Goose) are only +detected. A prerequisite the harness cannot provide is ``BLOCKED`` with the +exact remediation and blocks the layers that need it; it is never a pass. +``setup --check`` (``provision=False``) changes nothing and reports ``MISSING`` +for what a provisioning run would do. +""" +from __future__ import annotations + +import hashlib +import json +import os +import platform +import re +import shutil +import signal +import socket +import subprocess +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Dict, List, Optional, Sequence, Tuple + +from . import real_agents, stack +from .paths import E2E_DIR, WORKSPACE_ROOT + +READY = "READY" +PROVISIONED = "PROVISIONED" +MISSING = "MISSING" +BLOCKED = "BLOCKED" + +#: Prerequisites of each layer, in check order (a later check may rely on an +#: earlier one, e.g. the image check needs a reachable daemon). +#: Every stack layer imports the producer-built agent release, so each needs +#: the native build Python (PyInstaller + server runtime). +LAYER_REQUIREMENTS: Dict[str, Tuple[str, ...]] = { + "backend": ("docker", "backend_image", "stack_ports", "native_python"), + "plugin": ("docker", "backend_image", "stack_ports", "native_python", "java"), + "ui": ("docker", "backend_image", "stack_ports", "native_python", "java", + "gui_session", "stale_ides"), + "agents": ("goose", "node", "codex_acp"), + "browser": ("docker", "backend_image", "stack_ports", "native_python", "node", + "website_deps", "browser_python"), +} + +#: Image label recording which Dockerfile/requirements a harness build used. +IMAGE_LABEL = "org.code4me.e2e.deps" + +#: Playwright pinned for the harness-owned browser venv (the version the +#: browser layer is verified with). +PLAYWRIGHT_VERSION = "1.63.0" + +BROWSER_VENV = E2E_DIR / ".browser-venv" +SETUP_LOG_DIR = E2E_DIR / ".cache/setup" +AGENT_BIN_DIR = E2E_DIR / ".cache/agents/bin" + +GOOSE_INSTALL_HINT = ( + "install Goose once (https://block.github.io/goose/docs/getting-started/installation, e.g. " + "`curl -fsSL https://github.com/block/goose/releases/download/stable/download_cli.sh " + "| CONFIGURE=false bash`) or set CODE4ME_E2E_GOOSE_EXECUTABLE" +) + + +@dataclass +class Check: + id: str + status: str + detail: str = "" + remediation: str = "" + + @property + def usable(self) -> bool: + return self.status in (READY, PROVISIONED) + + def to_dict(self) -> Dict[str, str]: + payload = {"id": self.id, "status": self.status, "detail": self.detail} + if self.remediation: + payload["remediation"] = self.remediation + return payload + + +@dataclass +class Prerequisites: + """Outcome of one :func:`prepare` call.""" + + layers: Tuple[str, ...] + checks: Dict[str, Check] = field(default_factory=dict) + #: Resolved executables the layers must use (e.g. the Codex adapter). + executables: Dict[str, str] = field(default_factory=dict) + + def blocking(self, layer: str) -> List[Check]: + return [self.checks[name] for name in LAYER_REQUIREMENTS.get(layer, ()) + if name in self.checks and not self.checks[name].usable] + + def blocked_reason(self, layer: str) -> str: + return "; ".join( + f"{check.id}: {check.remediation or check.detail}" for check in self.blocking(layer) + ) + + @property + def ready(self) -> bool: + return all(check.usable for check in self.checks.values()) + + def to_dict(self) -> Dict[str, object]: + return { + "ready": self.ready, + "checks": [check.to_dict() for check in self.checks.values()], + "layers": {layer: ("BLOCKED" if self.blocking(layer) else "READY") for layer in self.layers}, + } + + +# --------------------------------------------------------------------------- +# Small process helpers (patched in unit tests) +# --------------------------------------------------------------------------- + + +def _run(command: Sequence[str], *, timeout: float = 60, cwd: Optional[Path] = None, + env: Optional[Dict[str, str]] = None) -> Tuple[int, str]: + """Run a bounded command; ``(127, reason)`` when it cannot start.""" + try: + result = subprocess.run(list(command), cwd=cwd, env=env, capture_output=True, + text=True, timeout=timeout) + except FileNotFoundError as error: + return 127, str(error) + except subprocess.TimeoutExpired: + return 124, f"timed out after {int(timeout)}s" + except OSError as error: + return 126, str(error) + return result.returncode, (result.stdout or "") + (result.stderr or "") + + +def _run_logged(command: Sequence[str], log_name: str, *, timeout: float, cwd: Optional[Path] = None, + env: Optional[Dict[str, str]] = None) -> Tuple[int, Path]: + """Run a long provisioning command with its output in ``.cache/setup``.""" + SETUP_LOG_DIR.mkdir(parents=True, exist_ok=True) + log_path = SETUP_LOG_DIR / log_name + with log_path.open("w", encoding="utf-8") as log: + log.write("# " + " ".join(command) + "\n") + log.flush() + try: + child = subprocess.Popen(list(command), cwd=cwd, env=env, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) + except OSError as error: + log.write(f"could not start: {error}\n") + return 127, log_path + try: + return child.wait(timeout=timeout), log_path + except subprocess.TimeoutExpired: + log.write(f"\n# timed out after {int(timeout)}s\n") + return 124, log_path + finally: + # Never leave a detached build/install behind: an interrupted gate + # (Ctrl-C, an agent's tool timeout) must not race the next run. + if child.poll() is None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.wait() + + +def _say(message: str) -> None: + print(f"setup: {message}", file=sys.stderr) + + +def _workspace() -> Optional[Path]: + return WORKSPACE_ROOT + + +# --------------------------------------------------------------------------- +# Docker daemon and backend image +# --------------------------------------------------------------------------- + + +def _docker_reachable() -> Tuple[bool, str]: + rc, output = _run(["docker", "info", "--format", "{{.ServerVersion}}"], timeout=30) + return rc == 0, output.strip()[-300:] + + +def check_docker(provision: bool, **_: object) -> Check: + if not shutil.which("docker"): + return Check("docker", BLOCKED, "the docker CLI is not on PATH", + "install Docker Desktop, or Colima with the docker CLI and compose plugin " + "(`brew install colima docker docker-compose`)") + try: + compose = " ".join(stack.compose_command()) + except stack.StackError as error: + return Check("docker", BLOCKED, str(error), + "install the Docker Compose plugin (`docker compose`) or `docker-compose`") + ok, output = _docker_reachable() + if ok: + return Check("docker", READY, f"daemon {output or 'reachable'}; {compose}") + colima = shutil.which("colima") + profile = _colima_profile() if colima else None + if not colima or profile is None: + return Check("docker", BLOCKED, f"the Docker daemon is not reachable: {output}", + "start Docker Desktop or Colima (the harness only starts Colima when the " + "active docker context is a Colima one)") + command = [colima, "start"] + ([] if profile == "default" else [profile]) + if not provision: + shown = " ".join(["colima", *command[1:]]) + return Check("docker", MISSING, f"the Docker daemon is not reachable; setup will run `{shown}`") + _say(f"the Docker daemon is not reachable; starting Colima profile {profile}") + rc, log_path = _run_logged(command, "colima-start.log", timeout=900) + ok, output = _docker_reachable() + if ok: + return Check("docker", PROVISIONED, f"started Colima; daemon {output}") + return Check("docker", BLOCKED, f"`colima start` exited {rc}; the daemon is still unreachable", + f"inspect {log_path} and start Colima/Docker manually") + + +def _colima_profile() -> Optional[str]: + """The Colima profile behind the active docker endpoint, or ``None``. + + Starting Colima is only right when Docker is already pointed at it; on a + Docker Desktop host it would create a VM and switch the global context. + """ + host = os.environ.get("DOCKER_HOST", "") + if host: + match = re.search(r"/\.colima/([^/]+)/docker\.sock", host) + return match.group(1) if match else None + rc, output = _run(["docker", "context", "show"], timeout=15) + name = (output.strip().splitlines() or [""])[-1].strip() if rc == 0 else "" + if name == "colima": + return "default" + if name.startswith("colima-"): + return name[len("colima-"):] + return None + + +def _server_root() -> Optional[Path]: + root = _workspace() + return root / "code4me2-server" if root else None + + +def image_fingerprint(server: Path) -> str: + """Identity of the image's dependency inputs (Dockerfile + requirements).""" + digest = hashlib.sha256() + for name in ("Dockerfile.cpu", "requirements.txt"): + digest.update(name.encode()) + digest.update((server / name).read_bytes()) + return digest.hexdigest() + + +def _image_labels(image: str) -> Optional[Dict[str, str]]: + rc, output = _run(["docker", "image", "inspect", image, "--format", "{{json .Config.Labels}}"], + timeout=60) + if rc != 0: + return None + try: + labels = json.loads(output.strip().splitlines()[-1] or "null") + except (ValueError, IndexError): + return {} + return labels if isinstance(labels, dict) else {} + + +def _image_requirements(image: str) -> Optional[str]: + rc, output = _run(["docker", "run", "--rm", "--entrypoint", "cat", image, "/app/requirements.txt"], + timeout=120) + return output if rc == 0 else None + + +def image_freshness(image: str, server: Path) -> Tuple[Optional[bool], str]: + """``(None, why)`` when absent, else ``(fresh, why)``.""" + labels = _image_labels(image) + if labels is None: + return None, f"{image} is not present" + expected = image_fingerprint(server) + recorded = labels.get(IMAGE_LABEL) + if recorded: + if recorded == expected: + return True, f"{image} was built by the harness from the current Dockerfile.cpu/requirements.txt" + return False, f"{image} was built from an older Dockerfile.cpu/requirements.txt" + inside = _image_requirements(image) + if inside is None: + return False, f"could not read /app/requirements.txt from {image}" + if inside == (server / "requirements.txt").read_text(encoding="utf-8"): + return True, f"{image} carries the checkout's requirements.txt" + return False, f"{image} carries a different requirements.txt than the checkout" + + +def check_backend_image(provision: bool, *, scenario, **_: object) -> Check: + server = _server_root() + if server is None or not (server / "Dockerfile.cpu").is_file(): + return Check("backend_image", BLOCKED, "code4me2-server/Dockerfile.cpu was not found", + "run the harness from the Code4Me workspace (or set CODE4ME_E2E_WORKSPACE)") + image = scenario.stack.image + fresh, why = image_freshness(image, server) + if fresh: + return Check("backend_image", READY, why) + if not provision: + return Check("backend_image", MISSING, f"{why}; setup will build it from Dockerfile.cpu") + _say(f"{why}; building it (the first build downloads several GB and can take a while)") + started = time.monotonic() + rc, log_path = _run_logged( + ["docker", "build", "-f", "Dockerfile.cpu", "-t", image, + "--label", f"{IMAGE_LABEL}={image_fingerprint(server)}", "."], + "backend-image-build.log", timeout=3 * 3600, cwd=server, + ) + if rc: + return Check("backend_image", BLOCKED, f"docker build exited {rc}", + f"inspect {log_path}; fix the build and rerun") + return Check("backend_image", PROVISIONED, + f"built {image} in {int(time.monotonic() - started)}s ({why})") + + +def _port_free(port: int) -> bool: + """True unless something listens on the loopback port. + + A plain bind also fails on TIME_WAIT sockets left by the previous run's + connections; Docker's forwarder and the stub bind with SO_REUSEADDR, so + only a live listener (or a reuse-address bind failure) is a conflict. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.settimeout(1.0) + if probe.connect_ex(("127.0.0.1", port)) == 0: + return False + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + probe.bind(("127.0.0.1", port)) + except OSError: + return False + return True + + +def _port_owner(port: int) -> str: + if not shutil.which("lsof"): + return "" + rc, output = _run(["lsof", "-nP", f"-iTCP:{port}", "-sTCP:LISTEN"], timeout=15) + lines = [line for line in output.splitlines()[1:] if line.strip()] + return (" (" + " ".join(lines[0].split()[:2]) + ")") if lines else "" + + +def check_stack_ports(provision: bool, *, scenario, **_: object) -> Check: + ports = { + "stack.backend_port": scenario.stack.backend_port, + "stack.db_port": scenario.stack.db_port, + "stack.redis_port": scenario.stack.redis_port, + "stack.stub_port": scenario.stack.stub_port, + } + busy = {name: port for name, port in ports.items() if not _port_free(port)} + if not busy: + return Check("stack_ports", READY, "ports " + ", ".join(str(port) for port in ports.values()) + " are free") + compose_owned = {"stack.backend_port", "stack.db_port", "stack.redis_port"} + if set(busy) <= compose_owned: + try: + own = stack.is_up(scenario) + except Exception: # noqa: BLE001 - an unknown owner stays blocked + own = False + if own: + return Check("stack_ports", READY, + f"ports held by this harness's own compose project {scenario.stack.project_name}") + described = ", ".join(f"{name}={port}{_port_owner(port)}" for name, port in busy.items()) + return Check("stack_ports", BLOCKED, f"ports in use by another process: {described}", + "stop that process or choose free ports with --set stack._port=N") + + +# --------------------------------------------------------------------------- +# JDK, native build Python, GUI session, leftover IDEs +# --------------------------------------------------------------------------- + + +def check_java(provision: bool, **_: object) -> Check: + java_home = os.environ.get("JAVA_HOME") + java = str(Path(java_home) / "bin/java") if java_home else shutil.which("java") + if not java: + return Check("java", BLOCKED, "no java on PATH and JAVA_HOME is unset", + "install a JDK (17 or newer); Gradle provisions the plugin's own toolchain") + rc, output = _run([java, "-version"], timeout=30) + first = next((line for line in output.splitlines() if line.strip()), "") + if rc != 0: + return Check("java", BLOCKED, f"`java -version` failed: {first}", + "install a JDK (17 or newer) or point JAVA_HOME at one") + return Check("java", READY, first.strip()) + + +#: What the native builds import: PyInstaller, the agent runtime, and the +#: server's research package (release producer, proxy contracts). +_NATIVE_PROBE = "import PyInstaller, acp, openai, pydantic, research.study.agents.participant_release" + +_native_choice: Optional[Tuple[str, str]] = None + + +def native_python_candidates() -> List[Tuple[str, str]]: + """Interpreters for the native builds, in preference order.""" + override = os.environ.get("CODE4ME_E2E_PYTHON") + if override: + return [(override, "CODE4ME_E2E_PYTHON")] + server = _server_root() + found = [] + for candidate, label in (((server / ".venv/bin/python") if server else None, "code4me2-server/.venv"), + (E2E_DIR / ".venv/bin/python", "e2e/.venv")): + if candidate is not None and candidate.is_file(): + found.append((str(candidate), label)) + return found + + +def _native_probe(python: str) -> Tuple[bool, str]: + rc, output = _run([python, "-c", _NATIVE_PROBE], timeout=120, cwd=_server_root()) + return rc == 0, (output.strip().splitlines() or [""])[-1][:300] + + +def resolve_native_python() -> Optional[Tuple[str, str]]: + """The first candidate that imports every build input (cached per process).""" + global _native_choice + if _native_choice is None: + for python, source in native_python_candidates(): + if _native_probe(python)[0]: + _native_choice = (python, source) + break + return _native_choice + + +def check_native_python(provision: bool, **_: object) -> Check: + global _native_choice + remediation = ( + "let `python3 -m code4me_e2e setup` create e2e/.venv, or point CODE4ME_E2E_PYTHON at an " + "interpreter with packaging/requirements-{runtime,build}.lock and the editable server package" + ) + candidates = native_python_candidates() + for python, source in candidates: + ok, why = _native_probe(python) + if ok: + _native_choice = (python, source) + return Check("native_python", READY, f"{python} ({source})") + if source == "CODE4ME_E2E_PYTHON": + return Check("native_python", BLOCKED, f"{python} cannot import the build inputs: {why}", remediation) + tried = ", ".join(source for _, source in candidates) or "no server venv" + if not provision: + return Check("native_python", MISSING, f"{tried}: cannot import the build inputs; setup will create e2e/.venv") + return _provision_native_venv(remediation) + + +def _provision_native_venv(remediation: str) -> Check: + """``e2e/.venv`` pinned like the release build CI (runtime + build locks).""" + global _native_choice + root = _workspace() + server = root / "code4me2-server" if root else None + if server is None: + return Check("native_python", BLOCKED, "workspace not found", remediation) + venv = E2E_DIR / ".venv" + python = venv / "bin/python" + _say(f"creating {venv} for the native agent/proxy builds") + steps = [] + if not python.is_file(): + steps.append(([sys.executable, "-m", "venv", str(venv)], "native-venv.log", 300)) + steps += [ + ([str(python), "-m", "pip", "install", + "-r", str(server / "packaging/requirements-runtime.lock"), + "-r", str(server / "packaging/requirements-build.lock")], "native-venv-locks.log", 1800), + # Editable server only: `research` resolves from its src/. The proxy is + # bundled from its own directory; installing it would write egg-info + # into the plugin checkout. + ([str(python), "-m", "pip", "install", "--no-deps", "-e", str(server)], + "native-venv-editable.log", 600), + ] + for command, log_name, timeout in steps: + rc, log_path = _run_logged(command, log_name, timeout=timeout) + if rc: + return Check("native_python", BLOCKED, f"`{' '.join(command[1:4])} …` exited {rc}; see {log_path}", + remediation) + ok, why = _native_probe(str(python)) + if not ok: + return Check("native_python", BLOCKED, f"{python} still cannot import the build inputs: {why}", remediation) + _native_choice = (str(python), "e2e/.venv") + return Check("native_python", PROVISIONED, f"created {venv} (pinned runtime + build locks)") + + +def check_gui_session(provision: bool, **_: object) -> Check: + system = platform.system() + if system == "Darwin": + rc, output = _run(["launchctl", "managername"], timeout=15) + manager = output.strip() + if rc == 0 and manager == "Aqua": + return Check("gui_session", READY, + "macOS GUI session (the IDE layer must also run outside a command sandbox)") + return Check("gui_session", BLOCKED, f"no macOS GUI login session (launchctl manager: {manager or 'unknown'})", + "run the IDE layer from a logged-in desktop session, not over SSH") + if system == "Linux": + if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"): + return Check("gui_session", READY, f"display {os.environ.get('DISPLAY') or os.environ.get('WAYLAND_DISPLAY')}") + return Check("gui_session", BLOCKED, "DISPLAY is not set", + "start an X server (for example `Xvfb :99 & export DISPLAY=:99`) or use xvfb-run") + return Check("gui_session", BLOCKED, f"the IDE layer is not supported on {system}") + + +def _harness_ide_processes() -> List[Tuple[int, str]]: + """Sandbox IDEs started by this harness (their home is under e2e/runs).""" + marker = f"-Duser.home={E2E_DIR / 'runs'}" + rc, output = _run(["ps", "-ww", "-eo", "pid=,command="], timeout=30) + found = [] + for line in output.splitlines(): + if marker in line and "-Drobot-server.port=" in line: + pid, _, command = line.strip().partition(" ") + if pid.isdigit() and int(pid) != os.getpid(): + found.append((int(pid), command)) + return found + + +def check_stale_ides(provision: bool, **_: object) -> Check: + """A gate holds the workspace lock, so any harness IDE still running is a leftover.""" + stale = _harness_ide_processes() + if not stale: + return Check("stale_ides", READY, "no leftover harness IDE is running") + if not provision: + return Check("stale_ides", MISSING, f"{len(stale)} leftover harness IDE(s); setup will stop them") + for pid, _ in stale: + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.monotonic() + 20 + while time.monotonic() < deadline and _harness_ide_processes(): + time.sleep(1) + for pid, _ in _harness_ide_processes(): + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + return Check("stale_ides", PROVISIONED, f"stopped {len(stale)} leftover harness IDE(s)") + + +# --------------------------------------------------------------------------- +# Node, website dependencies, browser Python +# --------------------------------------------------------------------------- + + +def check_node(provision: bool, **_: object) -> Check: + node, npm = shutil.which("node"), shutil.which("npm") + if not node or not npm: + return Check("node", BLOCKED, "node and npm must be on PATH", + "install Node.js 18 or newer (22 is verified), e.g. `brew install node@22`") + rc, output = _run([node, "--version"], timeout=30) + match = re.match(r"v(\d+)", output.strip()) + if rc != 0 or not match or int(match.group(1)) < 18: + return Check("node", BLOCKED, f"node {output.strip() or 'unknown'} is too old or unusable", + "install Node.js 18 or newer (22 is verified)") + return Check("node", READY, f"node {output.strip()} ({node})") + + +def check_website_deps(provision: bool, **_: object) -> Check: + server = _server_root() + website = server / "src/website" if server else None + if website is None or not (website / "package.json").is_file(): + return Check("website_deps", BLOCKED, "src/website/package.json was not found") + modules = website / "node_modules" + if modules.is_dir(): + # Present dependencies are used as they are; a stale install surfaces as + # a failed website build with its log (browser-build.log). + return Check("website_deps", READY, f"{modules} present") + if not provision: + return Check("website_deps", MISSING, "src/website/node_modules is absent; setup will run `npm ci`") + npm = shutil.which("npm") + if not npm: + return Check("website_deps", BLOCKED, "npm is not on PATH", "install Node.js 18 or newer") + _say("installing the website dependencies (npm ci)") + rc, log_path = _run_logged([npm, "ci"], "website-npm-ci.log", timeout=1800, cwd=website) + if rc: + return Check("website_deps", BLOCKED, f"`npm ci` exited {rc}", f"inspect {log_path}") + return Check("website_deps", PROVISIONED, "installed src/website/node_modules (npm ci)") + + +def browser_python() -> Tuple[str, str]: + """The Playwright interpreter and where it came from.""" + override = os.environ.get("CODE4ME_E2E_BROWSER_PYTHON") + if override: + return override, "CODE4ME_E2E_BROWSER_PYTHON" + venv_python = BROWSER_VENV / "bin/python" + if venv_python.is_file(): + return str(venv_python), "e2e/.browser-venv" + return sys.executable, "harness interpreter" + + +_CHROMIUM_PROBE = ( + "from playwright.sync_api import sync_playwright\n" + "with sync_playwright() as p:\n" + " browser = p.chromium.launch(headless=True)\n" + " browser.close()\n" +) + + +def _chromium_ready(python: str) -> Tuple[bool, str]: + rc, output = _run([python, "-c", _CHROMIUM_PROBE], timeout=180) + return rc == 0, (output.strip().splitlines() or [""])[-1][:300] + + +def check_browser_python(provision: bool, **_: object) -> Check: + python, source = browser_python() + ok, why = _chromium_ready(python) + if ok: + os.environ.setdefault("CODE4ME_E2E_BROWSER_PYTHON", python) + return Check("browser_python", READY, f"Playwright Chromium launches with {python} ({source})") + remediation = ( + f"`python3 -m venv {BROWSER_VENV} && {BROWSER_VENV}/bin/python -m pip install " + f"playwright=={PLAYWRIGHT_VERSION} && {BROWSER_VENV}/bin/python -m playwright install chromium`" + ) + if source == "CODE4ME_E2E_BROWSER_PYTHON": + return Check("browser_python", BLOCKED, f"{python} cannot launch Playwright Chromium: {why}", remediation) + if not provision: + return Check("browser_python", MISSING, + f"{python} cannot launch Playwright Chromium; setup will provision {BROWSER_VENV}") + venv_python = BROWSER_VENV / "bin/python" + _say(f"provisioning {BROWSER_VENV} (Playwright {PLAYWRIGHT_VERSION} + Chromium)") + steps = [] + if not venv_python.is_file(): + steps.append(([sys.executable, "-m", "venv", str(BROWSER_VENV)], "browser-venv.log", 300)) + steps += [ + ([str(venv_python), "-m", "pip", "install", f"playwright=={PLAYWRIGHT_VERSION}"], + "browser-venv-pip.log", 900), + ([str(venv_python), "-m", "playwright", "install", "chromium"], "browser-chromium.log", 1800), + ] + for command, log_name, timeout in steps: + rc, log_path = _run_logged(command, log_name, timeout=timeout) + if rc: + return Check("browser_python", BLOCKED, f"`{' '.join(command[1:4])} …` exited {rc}; see {log_path}", + remediation) + ok, why = _chromium_ready(str(venv_python)) + if not ok: + return Check("browser_python", BLOCKED, f"Chromium still does not launch: {why}", remediation) + os.environ["CODE4ME_E2E_BROWSER_PYTHON"] = str(venv_python) + return Check("browser_python", PROVISIONED, f"created {BROWSER_VENV} with Playwright {PLAYWRIGHT_VERSION}") + + +# --------------------------------------------------------------------------- +# Real agents: host Goose, the plugin's vendored Codex ACP adapter +# --------------------------------------------------------------------------- + + +def check_goose(provision: bool, **_: object) -> Check: + result = real_agents.probe_identity("goose") + if result.passed and result.identity is not None: + identity = result.identity + return Check("goose", READY, f"{identity.path} ({identity.version or 'version unknown'}, {identity.source})") + return Check("goose", BLOCKED, result.detail or "no goose executable found", GOOSE_INSTALL_HINT) + + +def vendored_codex_dir() -> Optional[Path]: + root = _workspace() + return root / "code4me2/dev/codex-acp-proxy/codex-acp" if root else None + + +def _bundled_codex_binaries(adapter: Path) -> List[Path]: + return sorted(adapter.glob("node_modules/@openai/codex-*/vendor/*/bin/codex")) + + +def _adapter_build_stale(adapter: Path) -> bool: + built = adapter / "dist/index.js" + if not built.is_file(): + return True + newest = max((path.stat().st_mtime for path in (adapter / "src").rglob("*") if path.is_file()), default=0) + return newest > built.stat().st_mtime + + +def codex_wrapper(adapter: Path, node: str) -> Path: + """A harness-owned ``codex-acp`` launcher for the vendored adapter build.""" + AGENT_BIN_DIR.mkdir(parents=True, exist_ok=True) + wrapper = AGENT_BIN_DIR / "codex-acp" + body = ("#!/bin/sh\n" + "# Generated by code4me_e2e prereqs: the plugin's vendored Codex ACP adapter.\n" + f"exec \"{node}\" \"{adapter / 'dist/index.js'}\" \"$@\"\n") + if not wrapper.is_file() or wrapper.read_text(encoding="utf-8") != body: + wrapper.write_text(body, encoding="utf-8") + wrapper.chmod(0o755) + return wrapper + + +def check_codex_acp(provision: bool, *, executables: Dict[str, str], + write_wrapper: Optional[bool] = None, **_: object) -> Check: + """The Codex ACP executable: an explicit override, else the vendored adapter. + + ``write_wrapper`` (default: ``provision``) allows writing the harness-owned + launcher even in a no-provisioning identity probe; ``setup --check`` never + writes it. + """ + write_wrapper = provision if write_wrapper is None else write_wrapper + override = os.environ.get(real_agents.ENV_OVERRIDES["codex"]) + if override: + result = real_agents.probe_identity("codex") + if result.passed and result.identity is not None: + executables["codex"] = result.identity.path + return Check("codex_acp", READY, f"{result.identity.path} (CODE4ME_E2E_CODEX_EXECUTABLE)") + return Check("codex_acp", BLOCKED, result.detail, "point CODE4ME_E2E_CODEX_EXECUTABLE at a codex-acp executable") + adapter = vendored_codex_dir() + node, npm = shutil.which("node"), shutil.which("npm") + remediation = ("run `npm ci && npm run build` in code4me2/dev/codex-acp-proxy/codex-acp, " + "or set CODE4ME_E2E_CODEX_EXECUTABLE to a codex-acp executable") + if adapter is None or not (adapter / "package.json").is_file(): + return Check("codex_acp", BLOCKED, "the vendored Codex ACP adapter was not found in the plugin checkout", + remediation) + if not node or not npm: + return Check("codex_acp", BLOCKED, "node and npm must be on PATH to run the Codex ACP adapter", + "install Node.js 18 or newer") + needs_install = not (adapter / "node_modules").is_dir() + needs_build = _adapter_build_stale(adapter) + if needs_install or needs_build: + if not provision: + work = " and ".join(item for item, needed in (("npm ci", needs_install), ("npm run build", needs_build)) if needed) + return Check("codex_acp", MISSING, f"the vendored adapter needs {work}; setup will run it") + if needs_install: + _say("installing the vendored Codex ACP adapter (npm ci)") + rc, log_path = _run_logged([npm, "ci"], "codex-acp-npm-ci.log", timeout=1800, cwd=adapter) + if rc: + return Check("codex_acp", BLOCKED, f"`npm ci` exited {rc}; see {log_path}", remediation) + _say("building the vendored Codex ACP adapter (npm run build)") + rc, log_path = _run_logged([npm, "run", "build"], "codex-acp-build.log", timeout=900, cwd=adapter) + if rc or not (adapter / "dist/index.js").is_file(): + return Check("codex_acp", BLOCKED, f"`npm run build` exited {rc}; see {log_path}", remediation) + if not _bundled_codex_binaries(adapter): + return Check("codex_acp", BLOCKED, + "the adapter's bundled native codex binary is missing (macOS XProtect can remove it)", + "reinstall it with `npm ci` in code4me2/dev/codex-acp-proxy/codex-acp; see its SETUP.md") + version = json.loads((adapter / "package.json").read_text(encoding="utf-8") or "{}").get("version", "?") + described = f"vendored adapter {version} ({adapter / 'dist/index.js'})" + if not write_wrapper: + existing = AGENT_BIN_DIR / "codex-acp" + if existing.is_file() and str(adapter / "dist/index.js") in existing.read_text(encoding="utf-8"): + executables["codex"] = str(existing) + return Check("codex_acp", READY, f"{described} via {existing}") + return Check("codex_acp", MISSING, f"{described}; setup will write the launcher {existing}") + wrapper = codex_wrapper(adapter, node) + executables["codex"] = str(wrapper) + status = PROVISIONED if (needs_install or needs_build) else READY + return Check("codex_acp", status, f"{described} via {wrapper}") + + +# --------------------------------------------------------------------------- +# Orchestration +# --------------------------------------------------------------------------- + + +CHECKS: Dict[str, Callable[..., Check]] = { + "docker": check_docker, + "backend_image": check_backend_image, + "stack_ports": check_stack_ports, + "java": check_java, + "native_python": check_native_python, + "gui_session": check_gui_session, + "stale_ides": check_stale_ides, + "node": check_node, + "website_deps": check_website_deps, + "browser_python": check_browser_python, + "goose": check_goose, + "codex_acp": check_codex_acp, +} + +#: A check that cannot run meaningfully while these earlier checks are unusable. +_DEPENDS_ON = {"backend_image": ("docker",), "stack_ports": ("docker",), + "website_deps": ("node",), "codex_acp": ("node",)} + + +def prepare(scenario, layers: Sequence[str], *, provision: bool = True) -> Prerequisites: + """Check (and by default provision) every prerequisite of ``layers``.""" + result = Prerequisites(tuple(layers)) + order: List[str] = [] + for layer in layers: + for name in LAYER_REQUIREMENTS.get(layer, ()): + if name not in order: + order.append(name) + for name in order: + blockers = [dep for dep in _DEPENDS_ON.get(name, ()) + if dep in result.checks and not result.checks[dep].usable] + if blockers: + result.checks[name] = Check(name, BLOCKED, "not checked: " + ", ".join(blockers) + " is not usable", + result.checks[blockers[0]].remediation) + continue + try: + result.checks[name] = CHECKS[name](provision, scenario=scenario, executables=result.executables) + except Exception as error: # noqa: BLE001 - a crashing check is a blocked prerequisite + result.checks[name] = Check(name, BLOCKED, f"the check crashed: {error}") + return result + + +def print_table(prerequisites: Prerequisites, stream=None) -> None: + stream = stream or sys.stderr + width = max((len(name) for name in prerequisites.checks), default=4) + for check in prerequisites.checks.values(): + print(f"{check.id:<{width}} {check.status:<11} {check.detail}", file=stream) + if check.remediation and not check.usable: + print(f"{'':<{width}} {'':<11} -> {check.remediation}", file=stream) diff --git a/e2e/code4me_e2e/process.py b/e2e/code4me_e2e/process.py new file mode 100644 index 0000000..96988fe --- /dev/null +++ b/e2e/code4me_e2e/process.py @@ -0,0 +1,38 @@ +"""Bounded subprocesses; only terminate process groups created by this run.""" +from __future__ import annotations + +import os +import signal +import subprocess +from pathlib import Path + + +def stop(process: subprocess.Popen, timeout: int = 15) -> None: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + pass + # A Gradle launcher may exit before its IDE child. Kill any remaining + # members of the private session even when the launcher already exited. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=timeout) + + +def run(command: list[str], *, cwd: Path, log_path: Path, + env: dict | None = None, timeout: int = 1200) -> int: + with log_path.open("w", encoding="utf-8") as log: + log.write("# " + " ".join(command) + "\n") + log.flush() + child = subprocess.Popen(command, cwd=cwd, env=env, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) + try: + return child.wait(timeout=timeout) + finally: + stop(child) diff --git a/e2e/code4me_e2e/real_agents.py b/e2e/code4me_e2e/real_agents.py new file mode 100644 index 0000000..bb1a568 --- /dev/null +++ b/e2e/code4me_e2e/real_agents.py @@ -0,0 +1,562 @@ +"""Host detection and identity probe for participant-installed (BYOA) agents. + +Goose and Codex are ``BYOA_EXTERNAL``: the participant installs the executable, +so the harness may only ever run an agent it can *identify*. This module owns: + +* discovery (explicit/settings override, ``CODE4ME_E2E__EXECUTABLE``, the + release-declared command, ``PATH``, then the same known per-user install + locations the plugin's ``DefaultByoaAgentResolver`` documents); +* the identity probe (absolute path, size, sha256, first ``--version`` line); +* an isolated agent home + sanitized environment, so probing never consumes the + developer's personal agent configuration or ambient provider credentials; +* typed :class:`ProbeResult` outcomes. A missing or unusable prerequisite is + ``BLOCKED`` with a reason, never a pass and never an untyped crash. + +No third-party dependency; every subprocess is bounded and process-group killed. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import shutil +import subprocess +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +from .config import AgentProbeReason + +__all__ = [ + "FRAMEWORK_ARGS", + "FRAMEWORK_EXECUTABLES", + "ENV_OVERRIDES", + "AgentIdentity", + "ArgvPlan", + "DetectedExecutable", + "ProbeResult", + "build_env", + "candidate_names", + "classify_failure", + "detect", + "executable_identity", + "isolated_home", + "known_locations", + "plan_argv", + "probe_identity", + "redact_failure_text", +] + +#: Candidate executable names per framework, in preference order. For Codex the +#: ACP-speaking adapter is preferred: the bare Codex CLI in this workspace does +#: not speak ACP on stdio (``plan_argv`` blocks it explicitly rather than +#: hanging in an interactive session). +FRAMEWORK_EXECUTABLES: Dict[str, Tuple[str, ...]] = { + "goose": ("goose",), + "codex": ("codex-acp", "codex"), +} + +#: Default launch argv appended after the resolved executable. ``goose acp`` +#: runs Goose as an ACP agent server on stdio; ``codex-acp`` is already an ACP +#: adapter and takes no subcommand. +FRAMEWORK_ARGS: Dict[str, Tuple[str, ...]] = { + "goose": ("acp",), + "codex": (), +} + +#: Settings override environment variables, mirroring ``agent.executable``. +ENV_OVERRIDES: Dict[str, str] = { + "goose": "CODE4ME_E2E_GOOSE_EXECUTABLE", + "codex": "CODE4ME_E2E_CODEX_EXECUTABLE", +} + +#: Environment variables stripped before an agent launch: ambient provider +#: credentials must not silently authenticate a probe (R4). +_STRIPPED_ENV_PREFIXES = ( + "CODE4ME_", "OPENAI_", "ANTHROPIC_", "GOOSE_", "CODEX_", "GEMINI_", + "AZURE_", "AWS_", "HUGGING_FACE_", "OLLAMA_", +) + +#: Quota/limit evidence. Checked before auth because ``429`` often accompanies +#: an otherwise-valid credential. +_QUOTA_PATTERNS = ( + "quota", "rate limit", "rate_limit", "ratelimit", "too many requests", + "429", "usage limit", "insufficient credit", "out of credits", + "credit balance", "billing", "limit reached", "exceeded your current", + # The research gateway's 402 as Goose 1.51 reports it over ACP + # (``data.reason: credits_exhausted``, "add more credits") and as the + # gateway phrases it. + "credits_exhausted", "credits exhausted", "add more credits", + "insufficient_quota", "quota_exhausted", "budget is used up", +) +#: Authentication evidence. +_AUTH_PATTERNS = ( + "401", "403", "unauthorized", "unauthorised", "not logged in", + "not authenticated", "login required", "please log in", "please login", + "codex login", "goose configure", "api key", "api_key", "apikey", + "missing auth", "auth token", "token expired", "invalid token", + "credentials", "authentication required", "sign in", "log in", +) + +_LONG_OPAQUE = re.compile(r"\b[A-Za-z0-9_\-]{32,}\b") +_BEARER = re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._\-]+") +_KEYED_TOKEN = re.compile(r"(?i)\b(sk|pk|ghp|gho|xox[baprs])[-_]?[A-Za-z0-9_\-]{8,}") + + +# --------------------------------------------------------------------------- +# Discovery +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class DetectedExecutable: + """One resolved executable and where it was found.""" + + framework: str + path: str + source: str # override | env | command | path | known_location + + +def known_locations(home: Optional[str] = None) -> List[Path]: + """Documented per-user install locations, in discovery order. + + Mirrors ``ByoaAgentResolver.byoaKnownLocations()`` in the plugin so the + harness resolves the same host the participant's plugin would. + """ + base = Path(home) if home else Path.home() + locations = [ + base / ".local" / "bin", + base / ".cargo" / "bin", + base / ".npm-global" / "bin", + base / "go" / "bin", + base / ".local" / "share" / "goose" / "bin", + Path("/usr/local/bin"), + Path("/opt/homebrew/bin"), + Path("/usr/bin"), + ] + return locations + + +def candidate_names(framework: str, package: Optional[str] = None) -> Tuple[str, ...]: + """Candidate executable names for a framework or a declared package. + + A package the framework does not know falls back to its own name, exactly + like the plugin's resolver. + """ + if package and package.strip(): + normalized = package.strip().lower() + if normalized in FRAMEWORK_EXECUTABLES: + return FRAMEWORK_EXECUTABLES[normalized] + return (normalized,) + return FRAMEWORK_EXECUTABLES.get(framework, (framework,)) + + +def _is_executable(path: Path) -> bool: + return path.is_file() and os.access(path, os.X_OK) + + +def _resolve_candidate(candidate: str, path_env: Optional[str]) -> Optional[Tuple[Path, str]]: + """Locate one candidate: a path is used directly, a bare name is searched.""" + if not candidate or not candidate.strip(): + return None + candidate = candidate.strip() + if os.sep in candidate or (os.altsep and os.altsep in candidate) or os.path.isabs(candidate): + path = Path(candidate).expanduser() + return (path, "override") if _is_executable(path) else None + found = shutil.which(candidate, path=path_env) + if found: + path = Path(found) + if _is_executable(path): + return path, "path" + for base in known_locations(os.environ.get("HOME")): + path = base / candidate + if _is_executable(path): + return path, "known_location" + return None + + +def detect( + framework: str, + *, + executable: Optional[str] = None, + command: Optional[str] = None, + package: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, + path_env: Optional[str] = None, +) -> Optional[DetectedExecutable]: + """Resolve the executable for ``framework``, or ``None``. + + Discovery order (first match wins): explicit ``executable`` (settings + override), the framework's env override, the release-declared ``command``, + then each candidate file name on ``PATH`` and in the known locations. + """ + environment = os.environ if env is None else env + search_path = path_env if path_env is not None else environment.get("PATH") + + if executable and executable.strip(): + resolved = _resolve_candidate(executable, search_path) + if resolved is not None: + path, source = resolved + return DetectedExecutable(framework, str(path), "override" if source == "override" else source) + return None + + override_var = ENV_OVERRIDES.get(framework) + if override_var and environment.get(override_var, "").strip(): + resolved = _resolve_candidate(environment[override_var], search_path) + if resolved is not None: + path, _source = resolved + return DetectedExecutable(framework, str(path), "env") + return None + + if command and command.strip(): + resolved = _resolve_candidate(command, search_path) + if resolved is not None: + path, _source = resolved + return DetectedExecutable(framework, str(path), "command") + return None + + for candidate in candidate_names(framework, package): + resolved = _resolve_candidate(candidate, search_path) + if resolved is not None: + path, source = resolved + return DetectedExecutable(framework, str(path), source) + return None + + +# --------------------------------------------------------------------------- +# Identity +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class AgentIdentity: + """The exact executable a probe would launch.""" + + framework: str + path: str + size: int + sha256: str + version: Optional[str] + source: str + + def to_dict(self) -> Dict[str, Any]: + return { + "framework": self.framework, + "path": self.path, + "size": self.size, + "sha256": self.sha256, + "version": self.version, + "source": self.source, + } + + +@dataclass(frozen=True) +class ProbeResult: + """Typed outcome of a detection/identity/protocol probe. + + ``status`` is ``PASS`` only when every required check ran. ``BLOCKED`` + carries an :class:`AgentProbeReason` value; absent prerequisites, missing + authentication, quota exhaustion and protocol failures can never pass. + """ + + framework: str + status: str + reason: Optional[str] = None + detail: str = "" + identity: Optional[AgentIdentity] = None + checks: Tuple[Dict[str, Any], ...] = () + protocol: Mapping[str, Any] = field(default_factory=dict) + + @property + def passed(self) -> bool: + return self.status == "PASS" + + @classmethod + def ok( + cls, + framework: str, + identity: Optional[AgentIdentity] = None, + *, + checks: Sequence[Mapping[str, Any]] = (), + protocol: Optional[Mapping[str, Any]] = None, + ) -> "ProbeResult": + return cls( + framework=framework, + status="PASS", + identity=identity, + checks=tuple(dict(check) for check in checks), + protocol=dict(protocol or {}), + ) + + @classmethod + def blocked( + cls, + framework: str, + reason: AgentProbeReason, + detail: str, + *, + identity: Optional[AgentIdentity] = None, + checks: Sequence[Mapping[str, Any]] = (), + protocol: Optional[Mapping[str, Any]] = None, + ) -> "ProbeResult": + return cls( + framework=framework, + status="BLOCKED", + reason=reason.value, + detail=detail, + identity=identity, + checks=tuple(dict(check) for check in checks), + protocol=dict(protocol or {}), + ) + + def to_dict(self) -> Dict[str, Any]: + payload: Dict[str, Any] = { + "framework": self.framework, + "status": self.status, + "reason": self.reason, + "detail": self.detail, + "checks": [dict(check) for check in self.checks], + "protocol": dict(self.protocol), + } + if self.identity is not None: + payload["identity"] = self.identity.to_dict() + return payload + + +def executable_identity( + path: Path, + *, + framework: str, + source: str, + version_timeout: float = 5.0, +) -> AgentIdentity: + """Hash and identify an executable without launching a session.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return AgentIdentity( + framework=framework, + path=str(path.resolve()), + size=path.stat().st_size, + sha256=digest.hexdigest(), + version=capture_version(path, timeout=version_timeout), + source=source, + ) + + +def capture_version(path: Path, *, timeout: float = 5.0) -> Optional[str]: + """First non-blank line of `` --version``, or ``None``. + + Bounded and process-group killed: a wrapper that ignores ``--version`` and + starts a server cannot stall the probe. ``stdin`` is closed so a + stdio-reading process sees EOF and exits. + """ + try: + child = subprocess.Popen( + [str(path), "--version"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + except OSError: + return None + try: + output, _ = child.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + try: + os.killpg(child.pid, 9) + except ProcessLookupError: + pass + try: + child.communicate(timeout=5) + except subprocess.TimeoutExpired: + pass + return None + if child.returncode != 0: + # No ``--version`` support (e.g. an argparse usage error): no version, + # rather than a usage line recorded as one. + return None + for line in (output or "").splitlines(): + line = line.strip() + if line: + return line[:200] + return None + + +def probe_identity( + framework: str, + *, + executable: Optional[str] = None, + command: Optional[str] = None, + package: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, + path_env: Optional[str] = None, +) -> ProbeResult: + """Detect + identify the host agent; ``missing_binary`` when unusable.""" + found = detect( + framework, + executable=executable, + command=command, + package=package, + env=env, + path_env=path_env, + ) + if found is None: + searched = ", ".join(candidate_names(framework, package)) + return ProbeResult.blocked( + framework, + AgentProbeReason.MISSING_BINARY, + f"no {framework} executable found (searched: {searched}); install it or set " + f"{ENV_OVERRIDES.get(framework, 'the agent executable override')}", + protocol={"searched": searched}, + ) + path = Path(found.path) + if not path.is_file() or not os.access(path, os.X_OK): + return ProbeResult.blocked( + framework, + AgentProbeReason.MISSING_BINARY, + f"{path} is not an executable file", + ) + identity = executable_identity(path, framework=framework, source=found.source) + return ProbeResult.ok(framework, identity) + + +# --------------------------------------------------------------------------- +# Launch plan +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ArgvPlan: + """The exact argv a probe would spawn, or a typed block.""" + + argv: List[str] + blocked_reason: Optional[AgentProbeReason] = None + blocked_detail: str = "" + + @property + def blocked(self) -> bool: + return self.blocked_reason is not None + + +def plan_argv( + framework: str, + identity: AgentIdentity, + configured: Optional[Sequence[str]] = None, +) -> ArgvPlan: + """Build the full argv; never a shell string. + + ``goose`` default is ``["", "acp"]``. For Codex, only an ACP adapter + (``codex-acp``) is launched by default: the bare CLI in this workspace does + not speak ACP on stdio, so launching it would hang in an interactive + session. An explicit ``configured`` argv (release ``agent_command_args``) + is always honored. + """ + if configured is not None: + args = [str(item) for item in configured] + else: + args = list(FRAMEWORK_ARGS.get(framework, ())) + argv = [identity.path, *args] + if not argv or not argv[0].strip(): + return ArgvPlan( + argv=[], + blocked_reason=AgentProbeReason.MISSING_BINARY, + blocked_detail="the resolved executable entry is empty", + ) + name = Path(identity.path).name.lower() + if framework == "codex" and configured is None and not name.startswith("codex-acp"): + return ArgvPlan( + argv=argv, + blocked_reason=AgentProbeReason.PROTOCOL, + blocked_detail=( + f"resolved {identity.path!r} is the Codex CLI, which does not speak ACP " + "on stdio; install codex-acp or point CODE4ME_E2E_CODEX_EXECUTABLE / " + "agent.executable at it" + ), + ) + return ArgvPlan(argv=argv) + + +# --------------------------------------------------------------------------- +# Isolated environment +# --------------------------------------------------------------------------- + + +def isolated_home(run_dir: Path, framework: str, override: Optional[str] = None) -> Path: + """Create the per-probe agent home; never the developer's own home.""" + home = Path(override).expanduser() if override else Path(run_dir) / "agent-home" / framework + if home.exists() and any(home.iterdir()): + raise ValueError("Agent home must be empty; use a fresh run directory instead of existing agent credentials") + if home.is_symlink(): + raise ValueError("Agent home must not be a symlink") + home.mkdir(mode=0o700, parents=True, exist_ok=True) + return home + + +def build_env( + home: Path, + *, + base_env: Optional[Mapping[str, str]] = None, + extra: Optional[Mapping[str, str]] = None, +) -> Dict[str, str]: + """Environment for an agent launch. + + Ambient provider/agent credentials are removed so an agent cannot be + silently authenticated by the developer's shell; ``HOME`` and the XDG + locations point at the isolated home. Values are never logged. + """ + source = os.environ if base_env is None else base_env + env = { + key: value + for key, value in source.items() + if not key.startswith(_STRIPPED_ENV_PREFIXES) + } + env["HOME"] = str(home) + env["CODEX_HOME"] = str(home / ".codex") + (home / ".codex").mkdir(parents=True, exist_ok=True) + env["XDG_CONFIG_HOME"] = str(home / ".config") + env["XDG_DATA_HOME"] = str(home / ".local" / "share") + env["XDG_CACHE_HOME"] = str(home / ".cache") + env["XDG_STATE_HOME"] = str(home / ".local" / "state") + env["NO_COLOR"] = "1" + env["TERM"] = "dumb" + if extra: + env.update({str(key): str(value) for key, value in extra.items()}) + return env + + +# --------------------------------------------------------------------------- +# Failure classification +# --------------------------------------------------------------------------- + + +def classify_failure(text: str) -> AgentProbeReason: + """Classify agent output into a typed blocked reason (fail closed). + + Quota evidence wins over auth (a ``429`` with a valid credential is not an + authentication failure); anything unrecognized is a protocol failure. + """ + lowered = (text or "").lower() + if any(pattern in lowered for pattern in _QUOTA_PATTERNS): + return AgentProbeReason.QUOTA + if any(pattern in lowered for pattern in _AUTH_PATTERNS): + return AgentProbeReason.MISSING_AUTH + return AgentProbeReason.PROTOCOL + + +def redact_failure_text(text: str, *, limit: int = 400) -> str: + """Bounded, best-effort redaction for a diagnostic snippet.""" + collapsed = " ".join((text or "").split()) + collapsed = _BEARER.sub(r"\1", collapsed) + collapsed = _KEYED_TOKEN.sub("", collapsed) + collapsed = _LONG_OPAQUE.sub("", collapsed) + return collapsed[:limit] + + +def describe(identity: Optional[AgentIdentity]) -> Dict[str, Any]: + """A log-safe identity summary (no environment values).""" + return asdict(identity) if identity is not None else {} diff --git a/e2e/code4me_e2e/report.py b/e2e/code4me_e2e/report.py new file mode 100644 index 0000000..fcb3aeb --- /dev/null +++ b/e2e/code4me_e2e/report.py @@ -0,0 +1,186 @@ +"""Run artifacts and the machine-readable report. + +Every run owns a directory under ``e2e/runs/-/`` containing: + +* ``state.json`` - resumable workflow state (ids, cookies, step results) +* ``report.json`` - the machine-readable report +* ``http.jsonl`` - one sanitized HTTP exchange per line (diagnostics) +* ``logs/backend.log`` - captured on failure +""" + +from __future__ import annotations + +import json +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +from . import stack +from .config import Scenario, sanitized_scenario + +RUNS_DIR_NEW = "runs" + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def new_run_id() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:6] + + +def runs_root(e2e_dir: Path) -> Path: + return e2e_dir / RUNS_DIR_NEW + + +def new_run_dir(e2e_dir: Path) -> Path: + run_dir = runs_root(e2e_dir) / new_run_id() + run_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + return run_dir + + +def latest_run_dir(e2e_dir: Path) -> Optional[Path]: + root = runs_root(e2e_dir) + if not root.is_dir(): + return None + candidates = sorted( + (path for path in root.iterdir() if (path / "state.json").is_file()), + key=lambda path: path.stat().st_mtime, + ) + return candidates[-1] if candidates else None + + +def write_state(run_dir: Path, state: Dict[str, Any]) -> None: + _write_json(run_dir / "state.json", state) + (run_dir / "state.json").chmod(0o600) + + +def read_state(run_dir: Path) -> Dict[str, Any]: + path = run_dir / "state.json" + if not path.is_file(): + return {} + return json.loads(path.read_text(encoding="utf-8")) + + +def append_http(run_dir: Path, records: List[Dict[str, Any]]) -> None: + path = run_dir / "http.jsonl" + with path.open("a", encoding="utf-8") as handle: + for record in records: + handle.write(json.dumps(record, sort_keys=True) + "\n") + + +def capture_backend_logs(scenario: Scenario, run_dir: Path, tail: int = 200) -> Optional[str]: + logs_dir = run_dir / "logs" + logs_dir.mkdir(parents=True, exist_ok=True) + path = logs_dir / "backend.log" + try: + text = stack.backend_logs(scenario, tail=tail) + except Exception as error: # noqa: BLE001 - diagnostics must never fail the run + text = f"(could not capture backend logs: {error})" + path.write_text(text, encoding="utf-8") + return str(path) + + +def _write_json(path: Path, payload: Dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True, default=str), encoding="utf-8") + + +def build_report( + *, + run_id: str, + started_at: str, + finished_at: str, + scenario: Scenario, + steps: List[Dict[str, Any]], + findings: List[Dict[str, Any]], + failed_step: Optional[str] = None, + blocked_by: Optional[str] = None, +) -> Dict[str, Any]: + report: Dict[str, Any] = { + "run_id": run_id, + "started_at": started_at, + "finished_at": finished_at, + "scenario": sanitized_scenario(scenario), + "steps": steps, + "failed_step": failed_step, + "blocked_by": blocked_by, + "findings": findings, + "next_actions": [], + } + report["next_actions"] = derive_next_actions(report) + return report + + +def derive_next_actions(report: Dict[str, Any]) -> List[str]: + actions: List[str] = [] + failing = next((step for step in report["steps"] if step["status"] == "FAIL"), None) + if failing is None and report.get("blocked_by"): + failing = next( + (step for step in report["steps"] if step["id"] == report["blocked_by"]), None + ) + if failing: + if failing.get("fix_hint"): + actions.append(failing["fix_hint"]) + actions.append( + "inspect the sanitized exchanges in report.json and logs/backend.log for this run" + ) + if report.get("findings"): + actions.append( + "review findings[] — these are application behaviors the harness refuses to hide" + ) + if not actions: + actions.append("all steps passed; the workflow is green") + return actions + + +def write_report(run_dir: Path, report: Dict[str, Any]) -> Path: + path = run_dir / "report.json" + _write_json(path, report) + return path + + +def print_summary(report: Dict[str, Any], as_json: bool) -> None: + if as_json: + summary = { + "run_id": report["run_id"], + "ok": report["failed_step"] is None and not report.get("blocked_by"), + "failed_step": report["failed_step"], + "blocked_by": report.get("blocked_by"), + "steps": [ + {"id": step["id"], "status": step["status"], "duration_ms": step["duration_ms"]} + for step in report["steps"] + ], + "findings": [finding.get("id") for finding in report.get("findings", [])], + "next_actions": report.get("next_actions", []), + } + if report.get("layers") is not None: + summary["layers"] = report["layers"] + if report.get("prerequisites") is not None: + summary["prerequisites"] = report["prerequisites"] + if report.get("report_path"): + summary["report_path"] = report["report_path"] + print(json.dumps(summary, indent=2, default=str)) + return + statuses = {step["id"]: step["status"] for step in report["steps"]} + width = max((len(step) for step in statuses), default=4) + for step_id, status in statuses.items(): + print(f"{step_id:<{width}} {status}") + print() + for layer in report.get("layers") or []: + reason = f" ({layer['reason']})" if layer.get("reason") else "" + print(f"layer {layer['layer']:<8} {layer['status']}{reason}") + if report.get("layers"): + print() + if report["failed_step"] or report.get("blocked_by"): + print(f"FAILED at {report['failed_step'] or report['blocked_by']}") + else: + print("ALL STEPS PASSED") + for action in report.get("next_actions", []): + print(f" -> {action}") + + +def monotonic_ms() -> int: + return int(time.monotonic() * 1000) diff --git a/e2e/code4me_e2e/runtime.py b/e2e/code4me_e2e/runtime.py new file mode 100644 index 0000000..3826969 --- /dev/null +++ b/e2e/code4me_e2e/runtime.py @@ -0,0 +1,158 @@ +"""Build current runtime sources without stamping versions or release resources. + +The native agent archive and the release manifest that describes it are produced +by the same module the real release pipeline uses +(``research.study.agents.participant_release native``). The harness therefore +imports a *tested* release -- manifest plus verified archive bytes -- instead of +inventing a digest, and the UI layer seeds those exact bytes into the IDE's +agent cache (the plugin bundles no agent). +""" +from __future__ import annotations + +import hashlib +import json +import platform +import shutil +import subprocess +from typing import TYPE_CHECKING, Tuple + +from . import process +from .paths import E2E_DIR, require_workspace + +if TYPE_CHECKING: + from pathlib import Path + + +def _fingerprint(paths: list[Path], root: Path) -> str: + digest = hashlib.sha256() + for base in paths: + for path in sorted(base.rglob("*.py") if base.is_dir() else [base]): + if "__pycache__" not in path.parts: + digest.update(str(path.relative_to(root)).encode()) + digest.update(path.read_bytes()) + return digest.hexdigest() + + +def _harness_python(server: Path) -> str: + """The native build interpreter the prerequisite check selects. + + CODE4ME_E2E_PYTHON, else the first of the server venv and the harness's + e2e/.venv that imports every build input. + """ + from .prereqs import resolve_native_python + + chosen = resolve_native_python() + if chosen is None: + raise RuntimeError("No native build Python imports PyInstaller and the server runtime: run " + "`python3 -m code4me_e2e setup` or set CODE4ME_E2E_PYTHON.") + return chosen[0] + + +def _platform_tag() -> Tuple[str, str]: + os_name = "macos" if platform.system() == "Darwin" else "linux" + arch = "arm64" if platform.machine().lower() in ("arm64", "aarch64") else "x64" + return os_name, arch + + +def _ensure_binary(server: Path, python: str, run_path: Path) -> Tuple[Path, str]: + """Build (cached by source fingerprint) the native agent binary.""" + root = require_workspace() + cache = E2E_DIR / ".cache/agent" + cache.mkdir(parents=True, exist_ok=True) + binary = cache / "dist/code4me2-agent/code4me2-agent" + fingerprint = _fingerprint([server / "src/code4me2_agent", server / "src/research", + server / "packaging/code4me2-agent.spec", + server / "packaging/requirements-runtime.lock", server / "pyproject.toml", + server / "packaging/archive_runtime.py"], root) + stamp = cache / "source.sha256" + if not binary.is_file() or not stamp.is_file() or stamp.read_text() != fingerprint: + rc = process.run([python, "-m", "PyInstaller", "--noconfirm", + "--distpath", str(cache / "dist"), "--workpath", str(cache / "build"), + "packaging/code4me2-agent.spec"], cwd=server, + log_path=run_path / "runtime-build.log") + if rc: + raise RuntimeError("Agent build failed; see runtime-build.log") + stamp.write_text(fingerprint) + return binary, fingerprint + + +def _produce_release(server: Path, python: str, binary: Path, fingerprint: str, + run_path: Path) -> Tuple[Path, Path]: + """Run the producer so the release manifest carries real test results.""" + cache = E2E_DIR / ".cache/agent" + release_dir = cache / "release" + os_name, arch = _platform_tag() + manifest_path = release_dir / f"native-{os_name}-{arch}.json" + stamp = cache / "release.sha256" + if not manifest_path.is_file() or not stamp.is_file() or stamp.read_text() != fingerprint: + shutil.rmtree(release_dir, ignore_errors=True) + # The producer refuses an existing output directory (immutable release), + # so only its parent is ensured here. + release_dir.parent.mkdir(parents=True, exist_ok=True) + # Version stamping is a release operation: reuse the version already + # embedded in the built executable. + version = subprocess.check_output([str(binary), "--version"], text=True, + timeout=30).strip().split()[-1] + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=server, + text=True).strip() + rc = process.run([python, "-m", "research.study.agents.participant_release", "native", + "--skip-build", "--version", version, + "--platform", f"{os_name}-{arch}", "--server-commit", commit, + "--bundle", str(binary.parent), "--output", str(release_dir)], + cwd=server, log_path=run_path / "runtime-release.log") + if rc or not manifest_path.is_file(): + raise RuntimeError("Release manifest production failed; see runtime-release.log") + stamp.write_text(fingerprint) + document = json.loads(manifest_path.read_text()) + return manifest_path, manifest_path.parent / document["artifacts"][0]["archive"] + + +def agent_release(run_path: Path) -> Tuple[Path, Path]: + """Return the producer manifest and its declared ZIP for the release import.""" + root = require_workspace() + server = root / "code4me2-server" + python = _harness_python(server) + binary, fingerprint = _ensure_binary(server, python, run_path) + return _produce_release(server, python, binary, fingerprint, run_path) + + +def seed_agent_cache(system_dir: Path, archive: Path) -> str: + """Place the study's agent archive in the IDE's verified installer cache. + + The plugin bundles no agent, and the e2e release has no public download URL, + so the archive the backend imports must already sit where the installer looks + (``/code4me/runtimes/code4me-agent/archives/.zip``). The plugin + re-verifies its size and digest before use. Returns that digest. + """ + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + target = system_dir / "code4me/runtimes/code4me-agent/archives" / f"{digest}.zip" + target.parent.mkdir(parents=True, exist_ok=True) + if not target.is_file() or hashlib.sha256(target.read_bytes()).hexdigest() != digest: + shutil.copyfile(archive, target) + return digest + + +def prepare(run_path: Path) -> list[str]: + """Cache native builds by source content, then supply Gradle staging inputs.""" + root = require_workspace() + server, plugin = root / "code4me2-server", root / "code4me2" + python = _harness_python(server) + _ensure_binary(server, python, run_path) + # The Gradle proxy builder has no up-to-date inputs. Cache it here using + # both the proxy and its shared server contracts. + cache = E2E_DIR / ".cache/agent" + proxy_stamp = cache / "proxy.sha256" + proxy_fingerprint = _fingerprint([plugin / "telemetry-acp-proxy/telemetry_acp_proxy", + plugin / "telemetry-acp-proxy/packaging/pyinstaller_entry.py", + plugin / "telemetry-acp-proxy/pyproject.toml", + plugin / "build.gradle.kts", server / "src/research"], root) + if not proxy_stamp.is_file() or proxy_stamp.read_text() != proxy_fingerprint or not (plugin / "telemetry-acp-proxy/dist").is_dir(): + rc = process.run([str(plugin / "gradlew"), "buildResearchProxyBundle", + f"-PresearchProxyPython={python}", "--console=plain"], cwd=plugin, + log_path=run_path / "proxy-build.log") + if rc: + raise RuntimeError("Proxy build failed; see proxy-build.log") + proxy_stamp.write_text(proxy_fingerprint) + # The plugin bundles no agent, like the participant build: the UI layer seeds + # the study's archive into the IDE cache instead (seed_agent_cache). + return ["--no-configuration-cache"] diff --git a/e2e/code4me_e2e/stack.py b/e2e/code4me_e2e/stack.py new file mode 100644 index 0000000..16351df --- /dev/null +++ b/e2e/code4me_e2e/stack.py @@ -0,0 +1,228 @@ +"""Lifecycle and DB helpers for the disposable e2e docker-compose stack. + +All docker interaction goes through ``docker compose -p -f ``. +The project name is always the scenario's (default ``code4me-e2e``), so this +module can never touch the developer stack's containers or volumes. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import time +from functools import lru_cache +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from .config import Scenario +from .paths import E2E_DIR + +COMPOSE_FILE = E2E_DIR / "docker-compose.e2e.yml" + + +class StackError(RuntimeError): + pass + + +@lru_cache(maxsize=1) +def compose_command() -> List[str]: + """Support Docker Desktop and Homebrew's standalone Compose installation.""" + for command in (["docker", "compose"], ["docker-compose"]): + if shutil.which(command[0]): + result = subprocess.run(command + ["version"], capture_output=True, timeout=30) + if result.returncode == 0: + return command + raise StackError("Docker Compose is required (docker compose or docker-compose).") + + +def compose_env(scenario: Scenario) -> Dict[str, str]: + """Interpolation variables passed to every ``docker compose`` invocation.""" + backend_image = scenario.stack.image + if backend_image == "code4me2-server-backend:latest": + # The tag is convenient for local builds but moves on every rebuild. + # Resolve it once per compose invocation so this E2E run uses immutable + # bytes even if another terminal retags latest while the suite runs. + inspected = subprocess.run( + ["docker", "image", "inspect", backend_image, "--format", "{{.Id}}"], + capture_output=True, + text=True, + timeout=30, + ) + image_id = inspected.stdout.strip() + if inspected.returncode == 0 and image_id.startswith("sha256:"): + backend_image = image_id + return { + "E2E_BACKEND_IMAGE": backend_image, + "E2E_BACKEND_PORT": str(scenario.stack.backend_port), + "E2E_DB_PORT": str(scenario.stack.db_port), + "E2E_REDIS_PORT": str(scenario.stack.redis_port), + "E2E_DB_PASSWORD": scenario.stack.db_password, + "E2E_DB_NAME": scenario.stack.db_name, + "E2E_DB_USER": scenario.stack.db_user, + "E2E_BOOTSTRAP_SIGNING_SECRET": scenario.stack.bootstrap_signing_secret, + "E2E_STUB_API_KEY": scenario.stack.stub_secret_value, + } + + +def compose( + scenario: Scenario, + args: List[str], + *, + check: bool = False, + timeout: int = 900, +) -> subprocess.CompletedProcess: + """Run one ``docker compose`` command for the scenario's project.""" + command = [ + *compose_command(), + "-p", + scenario.stack.project_name, + "-f", + str(COMPOSE_FILE), + *args, + ] + env = {**os.environ, **compose_env(scenario)} + result = subprocess.run( + command, + cwd=str(E2E_DIR), + env=env, + capture_output=True, + text=True, + timeout=timeout, + ) + if check and result.returncode != 0: + raise StackError( + "docker compose " + " ".join(args) + " failed:\n" + + (result.stderr or result.stdout) + ) + return result + + +def up(scenario: Scenario) -> str: + """Bring the stack up and wait until the backend schema is ready.""" + compose(scenario, ["up", "-d", "db", "redis"], check=True, timeout=900) + # A bind mount does not reload Python. Recreate just our backend so every + # invocation imports current source and applies new migrations. + result = compose(scenario, ["up", "-d", "--force-recreate", "backend"], check=True, timeout=900) + ready = wait_until_ready(scenario) + return result.stdout + ("\n" + ready if ready else "") + + +def down(scenario: Scenario) -> str: + """Remove only this project's containers and its named volume.""" + result = compose(scenario, ["down", "-v", "--remove-orphans"], timeout=300) + if result.returncode != 0: + raise StackError(result.stderr or result.stdout) + return result.stdout + result.stderr + + +def ps(scenario: Scenario) -> str: + result = compose(scenario, ["ps", "--format", "json"], timeout=120) + return result.stdout or result.stderr + + +def is_up(scenario: Scenario) -> bool: + result = compose(scenario, ["ps", "-q"], timeout=120) + return bool(result.stdout.strip()) + + +def backend_logs(scenario: Scenario, tail: int = 200) -> str: + result = compose(scenario, ["logs", "--tail", str(tail), "backend"], timeout=180) + return (result.stdout or "") + (result.stderr or "") + + +def psql(scenario: Scenario, sql: str, timeout: int = 60) -> str: + """Run one SQL statement in the disposable db service (trusted config only).""" + result = compose( + scenario, + [ + "exec", + "-T", + "db", + "psql", + "-U", + scenario.stack.db_user, + "-d", + scenario.stack.db_name, + "-t", + "-A", + "-c", + sql, + ], + timeout=timeout, + ) + if result.returncode != 0: + raise StackError(result.stderr.strip() or result.stdout.strip()) + return result.stdout.strip() + + +def try_psql(scenario: Scenario, sql: str, timeout: int = 30) -> Tuple[bool, str]: + try: + return True, psql(scenario, sql, timeout=timeout) + except (StackError, subprocess.SubprocessError) as error: + return False, str(error) + + +def ping(scenario: Scenario, timeout: float = 3.0) -> bool: + """HEAD /api/ping with no auth; True when the backend answers 200.""" + url = scenario.base_url + "/api/ping" + request = urllib.request.Request(url, method="HEAD") + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.status == 200 + except (urllib.error.URLError, TimeoutError, OSError, ValueError): + return False + + +def capabilities(scenario: Scenario, timeout: float = 5.0) -> Tuple[Optional[int], Any]: + """GET /api/acp/capabilities; returns ``(status, parsed_or_raw)``.""" + url = scenario.base_url + "/api/acp/capabilities" + try: + with urllib.request.urlopen(url, timeout=timeout) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + raw = error.read().decode("utf-8", "replace") + try: + return error.code, json.loads(raw) + except ValueError: + return error.code, raw + except (urllib.error.URLError, TimeoutError, OSError, ValueError): + return None, None + + +def wait_until_ready(scenario: Scenario, timeout: Optional[int] = None) -> str: + """Wait for liveness, then for ``schema_ready`` on /api/acp/capabilities.""" + deadline = time.monotonic() + ( + timeout if timeout is not None else scenario.timeouts.backend_ready_seconds + ) + last: str = "no response" + while time.monotonic() < deadline: + status, payload = capabilities(scenario) + if status == 200 and isinstance(payload, dict) and payload.get("schema_ready"): + return ( + f"backend ready: schema_revision={payload.get('schema_revision')} " + f"expected={payload.get('expected_schema_revision')}" + ) + if status == 200: + last = f"HTTP 200 but schema_ready=false: {payload}" + elif status is not None: + last = f"HTTP {status}: {payload}" + time.sleep(3) + raise StackError( + f"backend did not become schema-ready within " + f"{timeout or scenario.timeouts.backend_ready_seconds}s (last: {last})" + ) + + +def existing_containers() -> str: + """Names/ports of every running container (to prove the dev stack is safe).""" + result = subprocess.run( + ["docker", "ps", "--format", "{{.Names}}\t{{.Ports}}"], + capture_output=True, + text=True, + timeout=60, + ) + return result.stdout.strip() diff --git a/e2e/code4me_e2e/steps.py b/e2e/code4me_e2e/steps.py new file mode 100644 index 0000000..09626db --- /dev/null +++ b/e2e/code4me_e2e/steps.py @@ -0,0 +1,1201 @@ +"""The workflow steps for the code4me-e2e harness. + +Each ``step_*`` function drives the real HTTP API (plus one documented SQL +out-of-band step) and either returns a ``details`` dict or raises +:class:`StepFailure` / :class:`StepBlocked`. The workflow runner wraps them with +timing, persistence and report capture; steps stay plain functions on purpose. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +from . import runtime, stack +from .config import Scenario, redact +from .http import HttpClient, HttpError, redact_text +from .stub_provider import StubProvider + +#: Step ids in execution order. +STEP_ORDER = [ + "doctor", + "create_accounts", + "grant_roles", + "enable_researcher", + "provider_connection", + "register_release", + "qualify_release", + "create_profile", + "create_study", + "join_code", + "plugin_join", + "bootstrap", + "acp_prepare", + "send_message", + "telemetry", + "verify", + "revoke", +] + +STEP_DESCRIPTIONS = { + "doctor": "backend reachable, ACP schema ready, DB tables present, login", + "create_accounts": "create admin/researcher/participant accounts and log in", + "grant_roles": "promote the admin account via SQL (no admin-grant endpoint exists)", + "enable_researcher": "admin enables the researcher account (can_research)", + "provider_connection": "admin creates the stub provider connection", + "register_release": "admin imports the producer manifest and its verified archive", + "qualify_release": "the imported release is usable because its producer tests passed", + "create_profile": "researcher creates an agent profile pinned to the release", + "create_study": "researcher creates a study with a fixed agent profile", + "join_code": "researcher reads the study's join code", + "plugin_join": "participant resolves the website join code and accepts consent", + "bootstrap": "participant fetches and independently verifies the signed manifest", + "acp_prepare": "participant acquires session/project, mints an ACP grant, exchanges it", + "send_message": "agent creates a managed run and relays one inference turn", + "telemetry": "agent uploads a canonical telemetry batch", + "verify": "admin reads the batch receipt and the stored event row", + "revoke": "researcher revokes the enrollment; bootstrap access is refused", +} + + +class StepFailure(Exception): + """A step failed an assertion. Carries report-ready details.""" + + def __init__(self, message: str, *, details: Optional[Dict[str, Any]] = None, fix_hint: str = ""): + super().__init__(message) + self.message = message + self.details = details or {} + self.fix_hint = fix_hint + + +class StepBlocked(StepFailure): + """A prerequisite step did not complete, so this step cannot run.""" + + +@dataclass +class StepResult: + id: str + status: str + duration_ms: int + details: Dict[str, Any] = field(default_factory=dict) + fix_hint: str = "" + + def to_dict(self) -> Dict[str, Any]: + return { + "id": self.id, + "status": self.status, + "duration_ms": self.duration_ms, + "details": redact(self.details), + "fix_hint": self.fix_hint, + } + + +class Ctx: + """Per-run mutable context shared by steps.""" + + def __init__(self, scenario: Scenario, state: Dict[str, Any], run_dir: Path, *, fresh_login: bool = False): + self.scenario = scenario + self.state = state + self.run_dir = run_dir + self.clients: Dict[str, HttpClient] = {} + self.exchange_log: List[Dict[str, Any]] = [] + self.stub: Optional[StubProvider] = None + self.fresh_login = fresh_login + self.findings: List[Dict[str, str]] = list(state.get("findings") or []) + + # -- HTTP -------------------------------------------------------------- + + def client(self, role: str) -> HttpClient: + if role not in self.clients: + client = HttpClient( + self.scenario.base_url or "", + label=role, + timeout=float(self.scenario.timeouts.step_seconds), + on_record=self.exchange_log.append, + ) + cookies = self.state.get("cookies", {}).get(role) or {} + client.set_cookies(cookies) + self.clients[role] = client + if self.fresh_login and role in self.state.get("accounts", {}): + # Signing in through the IDE or fixture invalidates an earlier + # login. Resume with fresh credentials, not a stale cookie jar. + account = getattr(self.scenario, role) + auth = client.post("/api/user/authenticate", {"email": account.email, "password": account.password}) + if auth.status != 200: + raise StepFailure(f"Resuming {role} authentication failed (HTTP {auth.status})") + return self.clients[role] + + def snapshot_cookies(self) -> None: + """Persist the role cookie jars into state (needed for ``--from``).""" + self.state.setdefault("cookies", {}) + for role, client in self.clients.items(): + self.state["cookies"][role] = client.export_cookies() + + def records_since(self, index: int, limit: int = 6) -> List[Dict[str, Any]]: + return self.exchange_log[index:][-limit:] + + # -- stub provider ----------------------------------------------------- + + def ensure_stub(self) -> int: + if self.stub is None: + planned = int(self.state.get("stub_port") or self.scenario.stack.stub_port) + self.stub = StubProvider(token=self.state.get("stub_token")) + port = self.stub.start(planned, host=self.scenario.stack.stub_host) + self.state["stub_port"] = port + self.state["stub_token"] = self.stub.token + return int(self.stub.port or 0) + + @property + def stub_token(self) -> str: + return str(self.state.get("stub_token") or "") + + # -- state helpers ----------------------------------------------------- + + def require(self, *keys: str) -> None: + missing = [key for key in keys if not self.state.get(key)] + if missing: + raise StepBlocked( + "missing prerequisite state: " + ", ".join(missing), + details={"missing_state": missing}, + fix_hint="run the earlier steps first (or `run` without --from/--only)", + ) + + def get(self, key: str) -> Any: + return self.state.get(key) + + +# --------------------------------------------------------------------------- +# Small helpers +# --------------------------------------------------------------------------- + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _expect(resp, statuses, step: str, message: str, *, fix_hint: str = "") -> Any: + if resp.status not in statuses: + raise StepFailure( + f"{message} (HTTP {resp.status})", + details={ + "step": step, + "http_status": resp.status, + "response": resp.json if resp.json is not None else redact_text(resp.text)[:2000], + }, + fix_hint=_server_detail_hint(resp, fix_hint), + ) + return resp.json + + +def _server_detail_hint(resp, fallback: str) -> str: + """Prefer the server's own typed validation codes over a generic guess. + + A FastAPI ``{"detail": [{code, field, message}, ...]}`` body says exactly why + the request was refused; surfacing it keeps the report actionable instead of + blaming a prerequisite that did pass. + """ + payload = resp.json if isinstance(resp.json, dict) else None + detail = (payload or {}).get("detail") + entries = detail if isinstance(detail, list) else ([detail] if isinstance(detail, dict) else []) + codes = [] + for entry in entries: + if not isinstance(entry, dict): + continue + code = entry.get("code") or entry.get("type") + field = entry.get("field") + text = entry.get("message") + if code: + codes.append(f"{code}" + (f" at {field}" if field else "") + (f": {text}" if text else "")) + if codes: + return "server refused the request: " + "; ".join(codes[:4]) + return fallback + + +def _canonical_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str) + + +def _canonical_hash(value: Any) -> str: + return hashlib.sha256(_canonical_json(value).encode("utf-8")).hexdigest() + + +# --------------------------------------------------------------------------- +# 1. doctor +# --------------------------------------------------------------------------- + + +def step_doctor(ctx: Ctx) -> Dict[str, Any]: + scenario = ctx.scenario + anon = ctx.client("anon") + details: Dict[str, Any] = {"base_url": scenario.base_url} + + # Liveness. + try: + ping = anon.head("/api/ping") + except HttpError as error: + raise StepFailure( + f"backend is not reachable at {scenario.base_url}", + details={"error": str(error)}, + fix_hint="start the disposable stack: `python3 -m code4me_e2e stack up`", + ) + details["ping_status"] = ping.status + if ping.status != 200: + raise StepFailure( + f"/api/ping returned HTTP {ping.status}", + fix_hint="the backend process is up but unhealthy; check `stack up` logs", + ) + + # Readiness. + status, payload = stack.capabilities(scenario) + details["capabilities"] = payload if not isinstance(payload, str) else redact_text(payload) + if status != 200: + if status == 503 and isinstance(payload, dict): + detail = payload.get("detail") + raise StepFailure( + "the backend's managed-agent schema is not ready (schema drift)", + details={"status": status, "detail": detail, "capabilities": payload}, + fix_hint=( + "This backend points at a database whose alembic revision is not the " + f"code head ({detail!r}). The harness must provision its own clean DB: " + "run `python3 -m code4me_e2e stack down` then `stack up` (fresh volume), " + "or re-point --base-url at the harness stack. Do not reuse a drifted dev DB." + ), + ) + raise StepFailure( + f"/api/acp/capabilities returned HTTP {status}", + details={"status": status, "body": redact_text(str(payload))[:2000]}, + fix_hint="verify the backend started and migrations ran", + ) + if not payload.get("schema_ready"): + raise StepFailure( + "capabilities reported schema_ready=false", + details={"capabilities": payload}, + fix_hint="run `stack down && stack up` so the disposable DB is migrated to head", + ) + + # DB-level checks (best effort: only meaningful for the harness's own stack). + ok, out = stack.try_psql( + scenario, + "SELECT to_regclass('public.provider_connection') IS NOT NULL, " + + "to_regclass('public.agent_profile') IS NOT NULL, " + "(SELECT count(*) FROM config), " + "(SELECT count(*) FROM information_schema.columns " + " WHERE table_name='user' AND column_name='can_research'), " + "(SELECT count(*) FROM information_schema.columns " + " WHERE table_name='study' AND column_name='is_research');", + ) + if not ok: + details["db_probe"] = "skipped (no local db service for this project)" + else: + parts = [p.strip() for p in out.split("|")] + details["db_probe"] = { + "provider_connection": parts[0] == "t", + "agent_profile": parts[1] == "t", + "config_rows": int(parts[2]) if parts[2].isdigit() else 0, + "user_can_research": parts[3] == "1", + "study_is_research": parts[4] == "1", + } + expected = details["db_probe"] + if not ( + expected["provider_connection"] + and expected["agent_profile"] + and expected["config_rows"] >= 1 + and expected["user_can_research"] + and expected["study_is_research"] + ): + raise StepFailure( + "the disposable database is missing expected tables/columns", + details={"db_probe": expected}, + fix_hint=( + "the volume is half-initialised or schema-drifted: run " + "`python3 -m code4me_e2e stack down` then `stack up`" + ), + ) + + # Login check per account, only when the account already exists. + logins: Dict[str, Any] = {} + for role in ("admin", "researcher", "participant"): + account = getattr(scenario, role) + if ok: + exists = stack.psql( + scenario, + "SELECT count(*) FROM \"user\" WHERE email='" + + account.email.replace("'", "") + + "';", + ).strip() + else: + exists = "unknown" + if exists == "0": + logins[role] = "account not created yet (run will create it)" + continue + client = ctx.client(role) + resp = client.post( + "/api/user/authenticate", + {"email": account.email, "password": account.password}, + ) + if resp.status != 200: + raise StepFailure( + f"{role} login failed (HTTP {resp.status})", + details={"role": role, "http_status": resp.status}, + fix_hint=( + "check the account email/password in the scenario" + if ok + else ( + "this base-url is not the harness's own stack, so its accounts " + "are unknown; point --base-url at the harness stack " + "(http://localhost:) or supply credentials " + "that exist on this backend" + ) + ), + ) + logins[role] = "ok" + ctx.state[f"{role}_user_id"] = (resp.json or {}).get("user", {}).get("user_id") + ctx.snapshot_cookies() + details["logins"] = logins + return details + + +# --------------------------------------------------------------------------- +# 2. create_accounts +# --------------------------------------------------------------------------- + + +def step_create_accounts(ctx: Ctx) -> Dict[str, Any]: + scenario = ctx.scenario + config_id = ctx.get("config_id") + if not config_id: + try: + config_id = int(stack.psql(scenario, "SELECT config_id FROM config ORDER BY config_id LIMIT 1;")) + except (stack.StackError, ValueError) as error: + raise StepFailure( + "could not read a config_id from the database", + details={"error": str(error)}, + fix_hint="the disposable DB has no config row; run `stack down && stack up`", + ) + ctx.state["config_id"] = config_id + + results: Dict[str, Any] = {} + for role in ("admin", "researcher", "participant"): + account = getattr(scenario, role) + client = ctx.client(role) + created = client.post( + "/api/user/create", + { + "email": account.email, + "name": account.name, + "password": account.password, + "config_id": int(config_id), + }, + ) + if created.status not in (201, 409): + raise StepFailure( + f"creating the {role} account failed (HTTP {created.status})", + details={ + "role": role, + "http_status": created.status, + "response": created.json, + }, + fix_hint=( + "if this is HTTP 500 the backend could not contact the Celery broker; " + "the harness stack points CELERY_BROKER_* at its own redis" + ), + ) + auth = client.post( + "/api/user/authenticate", + {"email": account.email, "password": account.password}, + ) + if auth.status != 200: + raise StepFailure( + f"logging in as {role} failed (HTTP {auth.status})", + details={"role": role, "http_status": auth.status, "response": auth.json}, + fix_hint="check the account was created (HTTP 409 means it already existed)", + ) + user_id = ((auth.json or {}).get("user") or {}).get("user_id") + if not user_id: + raise StepFailure( + f"{role} login returned no user_id", + details={"response": auth.json}, + fix_hint="unexpected server response shape", + ) + ctx.state[f"{role}_user_id"] = user_id + results[role] = {"created": created.status == 201, "user_id": user_id} + ctx.snapshot_cookies() + return {"config_id": config_id, "accounts": results} + + +# --------------------------------------------------------------------------- +# 3. grant_roles +# --------------------------------------------------------------------------- + + +def step_grant_roles(ctx: Ctx) -> Dict[str, Any]: + scenario = ctx.scenario + ctx.require("admin_user_id") + email = scenario.admin.email.replace("'", "") + try: + stack.psql(scenario, f"UPDATE \"user\" SET is_admin=true WHERE email='{email}';") + except stack.StackError as error: + raise StepFailure( + "could not promote the admin account via SQL", + details={"error": str(error)}, + fix_hint="`docker compose exec db psql` failed; verify the stack is up", + ) + # Re-login so the session reflects the new privilege (defensive; the token + # only caches user_id, but this mirrors the plugin's login flow). + admin = ctx.client("admin") + auth = admin.post( + "/api/user/authenticate", + {"email": scenario.admin.email, "password": scenario.admin.password}, + ) + _expect(auth, (200,), "grant_roles", "admin re-login failed") + ctx.snapshot_cookies() + ctx.findings.append( + { + "id": "NO_ADMIN_GRANT_ENDPOINT", + "severity": "info", + "message": ( + "There is no admin-granting API. The harness promotes the admin " + "account with an out-of-band SQL UPDATE (the only such step)." + ), + "location": "code4me2-server/src/database/init.sql (is_admin default false)", + } + ) + ctx.state["findings"] = ctx.findings + return {"granted_admin": scenario.admin.email, "method": "SQL UPDATE \"user\" SET is_admin=true"} + + +# --------------------------------------------------------------------------- +# 4. enable_researcher +# --------------------------------------------------------------------------- + + +def step_enable_researcher(ctx: Ctx) -> Dict[str, Any]: + ctx.require("researcher_user_id") + admin = ctx.client("admin") + resp = admin.put( + f"/api/research/researchers/{ctx.get('researcher_user_id')}", + {"can_research": True}, + ) + payload = _expect( + resp, + (200,), + "enable_researcher", + "could not enable the researcher account", + fix_hint="the admin account must be promoted first (run grant_roles)", + ) + return {"researcher": payload.get("user")} + + +# --------------------------------------------------------------------------- +# 5. provider_connection +# --------------------------------------------------------------------------- + + +def step_provider_connection(ctx: Ctx) -> Dict[str, Any]: + scenario = ctx.scenario + ctx.require("researcher_user_id") + port = ctx.ensure_stub() + admin = ctx.client("admin") + label = "e2e-stub-" + ctx.run_dir.name + # Metered arms (Goose, built-in) fail closed without a price for the + # model, so the stub connection prices it (USD per million tokens). + body = { + "label": label, + "base_url": f"http://host.docker.internal:{port}/v1", + "secret_ref": scenario.stack.stub_secret_env, + "models": [scenario.agent.model], + "is_active": True, + "model_prices": { + scenario.agent.model: {"input_usd_per_million": "1", "output_usd_per_million": "4"} + }, + } + resp = admin.post("/api/research/provider-connections", body) + connection: Optional[dict] = None + if resp.status == 201: + connection = (resp.json or {}).get("connection") + elif resp.status == 409: + listing = admin.get("/api/research/provider-connections") + _expect(listing, (200,), "provider_connection", "could not list provider connections") + for item in (listing.json or {}).get("connections", []): + if item.get("label") == label: + connection = item + break + if connection is None: + raise StepFailure("provider connection label conflict but not found in list") + # An earlier run may have created the connection unpriced: (re)price it. + priced = admin.put(f"/api/research/provider-connections/{connection.get('connection_id')}", body) + if priced.status == 200: + connection = (priced.json or {}).get("connection") or connection + else: + _expect(resp, (201,), "provider_connection", "creating the provider connection failed") + connection_id = (connection or {}).get("connection_id") + if not connection_id: + raise StepFailure("provider connection returned no connection_id", details={"response": resp.json}) + ctx.state["connection_id"] = connection_id + + return { + "connection_id": connection_id, + "base_url": body["base_url"], + "stub_port": port, + } + + +# --------------------------------------------------------------------------- +# 6. register_release +# --------------------------------------------------------------------------- + + +def step_register_release(ctx: Ctx) -> Dict[str, Any]: + """Import the producer manifest and its archive through the byte-verifying API. + + The producer (``participant_release native``) built the archive and ran the + packaged executable's ``--self-check`` and ACP ``initialize``, so the manifest + already carries the platform test results. The harness imports exactly that + document; it never invents a digest or a test verdict. + """ + ctx.require("admin_user_id") + admin = ctx.client("admin") + manifest_path, archive_path = runtime.agent_release(ctx.run_dir) + document = manifest_path.read_text(encoding="utf-8") + artifact = (json.loads(document).get("artifacts") or [{}])[0] + resp = admin.post_multipart( + "/api/research/agents/releases/import", + fields={"manifest": document}, + files=[("archives", archive_path.name, archive_path.read_bytes())], + ) + payload = _expect( + resp, + (200, 201), + "register_release", + "importing the tested release failed", + fix_hint=( + "the manifest must declare the uploaded archive by basename and its " + "sha256/size must match the bytes; passing producer tests are required" + ), + ) + release = (payload or {}).get("release") or {} + release_id = release.get("release_id") + if not release_id: + raise StepFailure("release import returned no release_id", details={"response": payload}) + ctx.state["release_id"] = release_id + # Downstream steps (and the profile they create) must pin the imported + # release, not the synthetic scenario defaults. + ctx.scenario.agent.release_id = release_id + if artifact.get("sha256"): + ctx.scenario.agent.artifact_digest = artifact["sha256"] + return { + "release_id": release_id, + "created": (payload or {}).get("created"), + "artifact_digest": artifact.get("sha256"), + "verified_artifacts": len((payload or {}).get("verified_artifacts") or []), + } + + +# --------------------------------------------------------------------------- +# 7. qualify_release +# --------------------------------------------------------------------------- + + +def step_qualify_release(ctx: Ctx) -> Dict[str, Any]: + """Usability is derived from the imported producer tests; only verify it.""" + ctx.require("release_id") + admin = ctx.client("admin") + release_id = ctx.get("release_id") + fetched = admin.get(f"/api/research/agents/releases/{release_id}") + payload = _expect(fetched, (200,), "qualify_release", "fetching the release failed") + derived = ((payload or {}).get("release") or {}).get("status") + if derived != "QUALIFIED": + raise StepFailure( + f"release derived status is {derived!r}, expected 'QUALIFIED'", + details={"release": (payload or {}).get("release")}, + fix_hint=( + "the imported manifest must carry passing self_check and " + "acp_initialize results for the scenario platform" + ), + ) + return {"release_id": release_id, "derived_status": derived} + + +# --------------------------------------------------------------------------- +# 8. create_profile +# --------------------------------------------------------------------------- + + +def step_create_profile(ctx: Ctx) -> Dict[str, Any]: + scenario = ctx.scenario + ctx.require("researcher_user_id", "connection_id", "release_id") + researcher = ctx.client("researcher") + name = scenario.agent.profile_name + body = { + "name": name, + "model": scenario.agent.model, + "framework_version": scenario.agent.framework_version, + "connection_id": ctx.get("connection_id"), + "release_id": ctx.get("release_id"), + "tools_json": json.dumps(scenario.agent.tools), + "approval_policy": scenario.agent.approval_policy, + "max_steps": scenario.agent.max_steps, + "is_active": True, + } + resp = researcher.post("/api/agent/profiles", body) + profile: Optional[dict] = None + if resp.status == 201: + profile = (resp.json or {}).get("profile") + elif resp.status == 409: + listing = researcher.get("/api/agent/profiles") + for item in (listing.json or {}).get("profiles", []): + if item.get("name") == name: + profile = item + break + else: + _expect(resp, (201,), "create_profile", "creating the agent profile failed") + profile_id = (profile or {}).get("profile_id") + if not profile_id: + raise StepFailure("agent profile returned no profile_id", details={"response": resp.json}) + ctx.state["profile_id"] = profile_id + return {"profile_id": profile_id, "created": resp.status == 201, "owner_user_id": (profile or {}).get("owner_user_id")} + + +# --------------------------------------------------------------------------- +# 9. create_study +# --------------------------------------------------------------------------- + + +def step_create_study(ctx: Ctx) -> Dict[str, Any]: + """Create the current study configuration (draft/revision API was removed).""" + scenario = ctx.scenario + ctx.require("profile_id") + now = datetime.now(timezone.utc) + response = ctx.client("researcher").post("/api/research/studies", { + "name": scenario.study.name, + "description": scenario.study.description, + "profile_ids": [ctx.get("profile_id")], + # Required for metered arms (the stub model is priced by the + # provider_connection step); ignored for a Codex-only selection. + "default_budget_usd": "25", + "starts_at": scenario.study.start_at or (now - timedelta(hours=1)).isoformat(), + "ends_at": scenario.study.end_at or (now + timedelta(days=365)).isoformat(), + "telemetry_policy": {"allowed_field_classes": ["STRUCTURAL", "METRICS"]}, + "session_policy": {"idle_timeout_seconds": 900, "resume_grace_seconds": 300, "heartbeat_seconds": 5}, + }) + payload = _expect(response, (201,), "create_study", "creating the study failed") + study = payload.get("study") or {} + if not study.get("study_id") or not study.get("join_code"): + raise StepFailure("study response must contain study_id and join_code") + ctx.state["study_id"] = study["study_id"] + ctx.state["join_code"] = study["join_code"] + return {"study_id": study["study_id"], "status": study.get("research_status")} + + +def step_join_code(ctx: Ctx) -> Dict[str, Any]: + ctx.require("study_id") + response = ctx.client("researcher").get(f"/api/research/studies/{ctx.get('study_id')}") + payload = _expect(response, (200,), "join_code", "reading the study join code failed") + code = (payload.get("study") or {}).get("join_code") + if not code: + raise StepFailure("study has no join code") + ctx.state["join_code"] = code + return {"study_id": ctx.get("study_id"), "join_code_present": True} + + +def step_plugin_join(ctx: Ctx) -> Dict[str, Any]: + ctx.require("join_code", "participant_user_id") + participant = ctx.client("participant") + code = ctx.get("join_code") + + resolved = participant.get(f"/api/research/join/{code}") + resolved_payload = _expect( + resolved, (200,), "plugin_join", "resolving the join code failed (mirrors ResearchJoinCodeResolver)" + ) + me = participant.get("/api/research/participants/me") + _expect(me, (200,), "plugin_join", "fetching /api/research/participants/me failed") + + redeem = participant.post("/api/research/join", {"join_code": code, "accept_consent": True}) + redeem_payload = _expect( + redeem, + (200, 201), + "plugin_join", + "redeeming the join code failed (mirrors ResearchJoinCodeResolver.redeem)", + fix_hint="the participant must not already be enrolled in a different live study", + ) + enrollment_id = (redeem_payload or {}).get("enrollment_id") + if not enrollment_id: + raise StepFailure("join response had no enrollment_id", details={"response": redeem_payload}) + ctx.state["enrollment_id"] = enrollment_id + + status_resp = participant.get("/api/research/participants/me") + enrollments = (status_resp.json or {}).get("enrollments", []) + match = next((e for e in enrollments if e.get("enrollment_id") == enrollment_id), None) + if not match or match.get("status") != "ACTIVE": + raise StepFailure( + "enrollment is not ACTIVE after redeeming the join code", + details={"enrollment": match, "enrollments": enrollments}, + fix_hint="consent acceptance must activate the enrollment", + ) + ctx.state["revocation_epoch_before"] = match.get("revocation_epoch") + return { + "study_id": (resolved_payload or {}).get("study", {}).get("study_id"), + "enrollment_id": enrollment_id, + "status": match.get("status"), + } + + +# --------------------------------------------------------------------------- +# 12. bootstrap +# --------------------------------------------------------------------------- + + +def _verify_manifest_signature(manifest: Dict[str, Any], secret: str) -> Dict[str, Any]: + payload = { + key: value + for key, value in manifest.items() + if key not in ("manifest_digest", "signature") + } + digest = _canonical_hash(payload) + signature = hmac.new( + secret.encode("utf-8"), digest.encode("utf-8"), hashlib.sha256 + ).hexdigest() + return { + "recomputed_digest": digest, + "digest_matches": digest == manifest.get("manifest_digest"), + "recomputed_signature": signature, + "signature_matches": hmac.compare_digest(signature, str(manifest.get("signature") or "")), + } + + +def step_bootstrap(ctx: Ctx) -> Dict[str, Any]: + scenario = ctx.scenario + ctx.require("enrollment_id", "participant_user_id") + participant = ctx.client("participant") + environment = { + "os": scenario.platform.os, + "arch": scenario.platform.arch, + "ide_build": scenario.platform.ide_build, + "plugin_version": scenario.platform.plugin_version, + "host_kind": scenario.platform.host_kind, + } + resp = participant.post( + "/api/research/bootstrap/research-sessions", + { + "enrollment_id": ctx.get("enrollment_id"), + # Opaque execution context (phase 03): required for per-context + # session idempotency; mirrors the IntelliJ plugin's project context. + "context_id": "e2e-bootstrap-context", + "environment": environment, + }, + ) + payload = _expect( + resp, + (201,), + "bootstrap", + "fetching the signed bootstrap manifest failed (mirrors HttpBootstrapTransport)", + fix_hint=( + "bootstrap requires an ACTIVE enrollment, a live ACTIVE study, a " + "QUALIFIED release and a platform artifact for the configured platform" + ), + ) + manifest = (payload or {}).get("manifest") + if not isinstance(manifest, dict): + raise StepFailure("bootstrap response had no manifest object", details={"response": payload}) + + manifest_digest = manifest.get("manifest_digest") or (payload or {}).get("manifest_digest") + signature = manifest.get("signature") or (payload or {}).get("signature") + capability = manifest.get("session_capability") or {} + checks: Dict[str, Any] = { + "manifest_digest_is_sha256": bool( + isinstance(manifest_digest, str) + and len(manifest_digest) == 64 + and all(ch in "0123456789abcdef" for ch in manifest_digest) + ), + "signature_present": bool(signature), + "capability_scope": capability.get("scope"), + "capability_has_telemetry_write": "telemetry:write" in (capability.get("scope") or []), + } + if not checks["manifest_digest_is_sha256"]: + raise StepFailure( + "manifest_digest is not a 64-char sha256 hex string", + details={"manifest_digest": manifest_digest}, + ) + if not checks["signature_present"]: + raise StepFailure("manifest signature is missing") + if not checks["capability_has_telemetry_write"]: + raise StepFailure( + "session capability does not include the telemetry:write scope", + details={"scope": capability.get("scope")}, + ) + + verification = _verify_manifest_signature(manifest, scenario.stack.bootstrap_signing_secret) + checks.update(verification) + if not verification["digest_matches"]: + raise StepFailure( + "recomputed manifest digest does not match the server's", + details={"verification": verification}, + fix_hint="canonical JSON rules differ from research/canonical.py", + ) + if not verification["signature_matches"]: + raise StepFailure( + "manifest HMAC signature does not verify with the configured signing secret", + details={"verification": verification}, + fix_hint="check stack.bootstrap_signing_secret matches the backend env", + ) + + research_session = manifest.get("research_session") or {} + ctx.state["research_session_id"] = research_session.get("research_session_id") + ctx.state["session_capability"] = capability + ctx.state["manifest_digest"] = manifest_digest + return { + "research_session_id": ctx.get("research_session_id"), + "assignment": manifest.get("assignment"), + "agent_release": manifest.get("agent_release"), + "checks": checks, + } + + +# --------------------------------------------------------------------------- +# 13. acp_prepare +# --------------------------------------------------------------------------- + + +def step_acp_prepare(ctx: Ctx) -> Dict[str, Any]: + scenario = ctx.scenario + ctx.require("participant_user_id") + participant = ctx.client("participant") + + acquire = participant.get("/api/session/acquire") + _expect(acquire, (200,), "acp_prepare", "session acquire failed") + ctx.snapshot_cookies() + + project = participant.post("/api/project/create", {"project_name": "code4me-e2e"}) + project_payload = _expect(project, (201,), "acp_prepare", "creating the project failed") + project_id = (project_payload or {}).get("project_token") + if not project_id: + raise StepFailure("project creation returned no project_token", details={"response": project_payload}) + ctx.state["project_id"] = project_id + ctx.snapshot_cookies() + + activate = participant.put("/api/project/activate", {"project_id": project_id}) + _expect(activate, (200,), "acp_prepare", "activating the project failed") + + workspace = "/tmp/code4me-e2e-workspace" + grant_resp = participant.post( + "/api/acp/grant", + { + "project_id": project_id, + "workspace": workspace, + "launch_id": "e2e-launch-1", + "path_format": "posix", + "managed_protocol_version": "1", + }, + ) + grant_payload = _expect( + grant_resp, + (201,), + "acp_prepare", + "minting an ACP launch grant failed", + fix_hint="the plugin must hold an active session and project association", + ) + grant = (grant_payload or {}).get("grant") + if not grant: + raise StepFailure("grant response had no grant", details={"response": grant_payload}) + + exchange = participant.post("/api/acp/session/exchange", {"grant": grant}) + exchange_payload = _expect( + exchange, (200,), "acp_prepare", "exchanging the ACP grant failed" + ) + acp_token = (exchange_payload or {}).get("acp_token") + if not acp_token: + raise StepFailure("exchange response had no acp_token", details={"response": exchange_payload}) + ctx.state["acp_token"] = acp_token + + readiness = participant.get("/api/acp/readiness") + ready_payload = _expect( + readiness, + (200,), + "acp_prepare", + "ACP readiness probe failed", + fix_hint=( + "readiness resolves the participant's sticky assignment: needs an ACTIVE " + "enrollment, a live study and an active profile" + ), + ) + return { + "project_id": project_id, + "workspace": (grant_payload or {}).get("workspace"), + "readiness": ready_payload, + } + + +# --------------------------------------------------------------------------- +# 14. send_message +# --------------------------------------------------------------------------- + + +def step_send_message(ctx: Ctx) -> Dict[str, Any]: + scenario = ctx.scenario + ctx.require("acp_token", "research_session_id") + participant = ctx.client("participant") + bearer = ctx.get("acp_token") + run_id = str(uuid.uuid4()) + session_id = ctx.get("research_session_id") + ctx.state["run_id"] = run_id + + # Name the research session explicitly: another client of the same account + # (the plugin layer's Kotlin fixture) can hold an active session too, and the + # server never guesses between them (RESEARCH_CONTEXT_AMBIGUOUS). + created = participant.post( + "/api/acp/runs", + {"run_id": run_id, "session_id": session_id, "research_session_id": session_id}, + bearer=bearer, + ) + created_payload = _expect( + created, + (200, 201), + "send_message", + "creating the managed run failed", + fix_hint=( + "the ACP scope user must resolve an active study agent profile " + "(check /api/acp/readiness)" + ), + ) + + # A resumed run (`--from send_message`) is a fresh process: the in-process + # stub server from `provider_connection` is gone, so start it again on the + # persisted port before the relay tries to reach it. + ctx.ensure_stub() + expected = scenario.message.expected_substring or ctx.stub_token + if not expected: + raise StepFailure( + "no expected answer token is configured", + fix_hint="start the stub provider (provider_connection step) or set message.expected_substring", + ) + inference = participant.post( + "/api/acp/inference", + { + "run_id": run_id, + "session_id": session_id, + "request": {"messages": [{"role": "user", "content": scenario.message.prompt}]}, + }, + bearer=bearer, + ) + _expect( + inference, + (200,), + "send_message", + "the managed inference relay failed", + fix_hint=( + "the provider connection must be active and its secret_ref env var present; " + "the stub must be reachable at host.docker.internal: " + "(on a Linux Docker engine run with --set stack.stub_host=0.0.0.0)" + ), + ) + if expected not in (inference.text or ""): + raise StepFailure( + f"assistant answer did not contain the expected token {expected!r}", + details={ + "expected_substring": expected, + "response": inference.json, + "stub_requests": (ctx.stub.requests if ctx.stub else []), + }, + fix_hint="the inference relay did not reach the stub provider, or the response shape changed", + ) + # Exact request construction at the relay boundary: the provider must receive + # a non-streaming Chat Completions request carrying the participant messages + # and the policy model, with no client-supplied override. The stub records + # only shape metadata, never prompt content. + forwarded = list(ctx.stub.requests) if ctx.stub else [] + last = forwarded[-1] if forwarded else {} + if not forwarded or last.get("message_count", 0) < 1 or last.get("stream") or not last.get("model"): + raise StepFailure( + "the relay forwarded an unexpected completion request", + details={"forwarded": forwarded}, + fix_hint=( + "the managed relay must forward the participant messages non-streaming " + "with the policy model and no client-side temperature/model override" + ), + ) + return { + "run_id": run_id, + "session_id": session_id, + "policy": (created_payload or {}).get("policy"), + "answer_contains_expected": True, + "forwarded_request": { + "message_count": last.get("message_count"), + "model": last.get("model"), + "stream": last.get("stream"), + }, + } + + +# --------------------------------------------------------------------------- +# 15. telemetry +# --------------------------------------------------------------------------- + + +def _canonical_event(ctx: Ctx) -> Dict[str, Any]: + return { + "event_id": str(uuid.uuid4()), + "schema_version": "1", + "event_type": "tool.started", + "source": "acp", + "study_id": ctx.get("study_id"), + "enrollment_id": ctx.get("enrollment_id"), + "research_session_id": ctx.get("research_session_id"), + "agent_run_id": ctx.get("run_id"), + "occurred_at": _now_iso(), + "monotonic_ns": None, + "emitter_id": "code4me-e2e-harness", + "emitter_sequence": 1, + "correlations": {}, + "lifecycle_state": "started", + "payload": {"status": "started"}, + "metrics": {}, + "privacy": {}, + "provenance": { + "source": "acp", + "normalizer_version": "e2e-harness-v1", + "fidelity": "normalized", + }, + "coverage": {"state": "AVAILABLE"}, + "unknown_event_type": None, + "unknown_source": None, + "unknown_lifecycle_state": None, + } + + +def step_telemetry(ctx: Ctx) -> Dict[str, Any]: + ctx.require( + "study_id", + "enrollment_id", + "research_session_id", + "session_capability", + ) + participant = ctx.client("participant") + event = _canonical_event(ctx) + batch_id = str(uuid.uuid4()) + body = { + "batch_id": batch_id, + "protocol_version": "1", + "telemetry_schema_version": "1", + "session_capability": ctx.get("session_capability"), + "events": [event], + "client_instance_id": "code4me-e2e-harness", + } + resp = participant.post("/api/research/telemetry/batches", body) + payload = _expect( + resp, + (200,), + "telemetry", + "telemetry batch submission failed", + fix_hint="the session capability must be bound to this enrollment/session", + ) + accepted_ids = [item.get("event_id") for item in (payload or {}).get("accepted", [])] + rejected = (payload or {}).get("rejected", []) + retryable = (payload or {}).get("retryable", []) + if event["event_id"] not in accepted_ids or rejected or retryable: + raise StepFailure( + "telemetry event was not accepted", + details={"response": payload}, + fix_hint="check the event context ids and capability scope (telemetry:write)", + ) + ctx.state["batch_id"] = batch_id + ctx.state["telemetry_receipt_id"] = (payload or {}).get("receipt_id") + ctx.state["telemetry_event_id"] = event["event_id"] + return { + "batch_id": batch_id, + "receipt_id": ctx.get("telemetry_receipt_id"), + "event_id": event["event_id"], + "accepted": accepted_ids, + } + + +# --------------------------------------------------------------------------- +# 16. verify +# --------------------------------------------------------------------------- + + +def step_verify(ctx: Ctx) -> Dict[str, Any]: + ctx.require("telemetry_receipt_id", "telemetry_event_id") + admin = ctx.client("admin") + resp = admin.get( + f"/api/research/telemetry/receipts/{ctx.get('telemetry_receipt_id')}" + ) + payload = _expect( + resp, + (200,), + "verify", + "reading the telemetry batch receipt failed", + ) + stored = None + try: + stored = stack.psql( + ctx.scenario, + "SELECT count(*) FROM research_event WHERE event_id='" + + str(ctx.get("telemetry_event_id")) + + "';", + ) + except stack.StackError as error: + stored = f"probe failed: {error}" + if stored != "1": + raise StepFailure( + "the telemetry event was not persisted in research_event", + details={"research_event_count": stored, "receipt": payload}, + fix_hint="ingestion committed the receipt but not the event; inspect backend logs", + ) + return { + "receipt_id": ctx.get("telemetry_receipt_id"), + "research_event_rows": stored, + "accepted": ((payload or {}).get("receipt") or {}).get("accepted"), + } + + +# --------------------------------------------------------------------------- +# 17. revoke +# --------------------------------------------------------------------------- + + +def step_revoke(ctx: Ctx) -> Dict[str, Any]: + ctx.require("enrollment_id", "study_id") + response = ctx.client("researcher").post( + f"/api/research/studies/{ctx.get('study_id')}/enrollments/{ctx.get('enrollment_id')}/revoke", {}) + payload = _expect(response, (200,), "revoke", "revoking the enrollment failed") + status = ctx.client("participant").get("/api/research/participants/me") + own = _expect(status, (200,), "revoke", "reading revoked enrollment failed") + match = next((e for e in own.get("enrollments", []) if e.get("enrollment_id") == ctx.get("enrollment_id")), {}) + if not payload.get("revoked") or match.get("status") != "REVOKED": + raise StepFailure("enrollment must become REVOKED", details={"enrollment": match}) + # A previously issued capability must no longer bootstrap this enrollment. + refused = ctx.client("participant").post("/api/research/bootstrap/research-sessions", { + "enrollment_id": ctx.get("enrollment_id"), "context_id": "e2e-after-revoke", + "environment": {"os": ctx.scenario.platform.os, "arch": ctx.scenario.platform.arch}, + }) + _expect(refused, (403, 409, 410), "revoke", "revoked enrollment was still usable") + return {"status": match["status"], "bootstrap_status": refused.status} + + +#: Registry the workflow runner uses. +STEPS = { + "doctor": step_doctor, + "create_accounts": step_create_accounts, + "grant_roles": step_grant_roles, + "enable_researcher": step_enable_researcher, + "provider_connection": step_provider_connection, + "register_release": step_register_release, + "qualify_release": step_qualify_release, + "create_profile": step_create_profile, + "create_study": step_create_study, + "join_code": step_join_code, + "plugin_join": step_plugin_join, + "bootstrap": step_bootstrap, + "acp_prepare": step_acp_prepare, + "send_message": step_send_message, + "telemetry": step_telemetry, + "verify": step_verify, + "revoke": step_revoke, +} diff --git a/e2e/code4me_e2e/stub_provider.py b/e2e/code4me_e2e/stub_provider.py new file mode 100644 index 0000000..8b575d2 --- /dev/null +++ b/e2e/code4me_e2e/stub_provider.py @@ -0,0 +1,154 @@ +"""A tiny OpenAI-compatible stub provider. + +The harness points an administrator-managed ``provider_connection`` at this +process so the real inference relay forwards a chat-completions request here. +The stub answers deterministically with the configured token, which lets the +workflow assert an exact answer end to end without any model or API key. + +It serves: + +* ``POST /v1/chat/completions`` — non-streaming OpenAI completion. +* ``GET /health`` — liveness. + +It ignores the ``Authorization`` header (the backend always sends the resolved +secret; the stub must not depend on it). +""" + +from __future__ import annotations + +import json +import threading +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, List, Optional + + +class StubProvider: + """Threaded HTTP server that echoes a deterministic assistant token.""" + + def __init__(self, token: Optional[str] = None) -> None: + self.token = token or f"E2E_STUB_ANSWER_{uuid.uuid4().hex}" + self.requests: List[Dict[str, Any]] = [] + self._server: Optional[ThreadingHTTPServer] = None + self._thread: Optional[threading.Thread] = None + self.port: Optional[int] = None + self._lock = threading.Lock() + #: Monotonic receipt for every accepted request. A caller records + #: :meth:`request_count` before an action and asserts the delta, so a + #: proof is per-turn instead of "the stub was reached at some point". + self._request_count = 0 + + # -- lifecycle --------------------------------------------------------- + + def start(self, port: int = 0, host: str = "127.0.0.1") -> int: + if self._server is not None: + return self.port or 0 + handler = _make_handler(self) + # A resumed provider connection still names this port. Moving silently + # would direct the backend to another process and invalidate the test. + # ``host`` only widens where it listens; the harness itself keeps + # calling it on loopback (``base_url``). + server = ThreadingHTTPServer((host, port), handler) + server.daemon_threads = True + self._server = server + self.port = server.server_address[1] + self._thread = threading.Thread( + target=server.serve_forever, name="code4me-e2e-stub", daemon=True + ) + self._thread.start() + return self.port + + def stop(self) -> None: + if self._server is not None: + self._server.shutdown() + self._server.server_close() + self._server = None + if self._thread is not None: + self._thread.join(timeout=5) + self._thread = None + + @property + def base_url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def record(self, entry: Dict[str, Any]) -> None: + with self._lock: + self._request_count += 1 + self.requests.append({**entry, "receipt": self._request_count}) + + def request_count(self) -> int: + """The latest monotonic receipt number (0 before any request).""" + with self._lock: + return self._request_count + + def requests_since(self, receipt: int) -> List[Dict[str, Any]]: + """Requests received after ``receipt`` (strictly greater).""" + with self._lock: + return [item for item in self.requests if int(item.get("receipt", 0)) > int(receipt)] + + def completion_body(self, requested_model: str) -> Dict[str, Any]: + return { + "id": "chatcmpl-e2e", + "object": "chat.completion", + "created": 0, + "model": requested_model or "e2e-stub-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": f"Stub answer: {self.token}", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 3, + "completion_tokens": 4, + "total_tokens": 7, + }, + } + + +def _make_handler(stub: StubProvider): + class _Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args: Any) -> None: # noqa: D401 - silence access log + return + + def _send(self, status: int, payload: Dict[str, Any]) -> None: + body = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 - http.server API + if self.path.split("?")[0] == "/health": + self._send(200, {"status": "ok", "token": stub.token}) + return + self._send(404, {"error": "not found"}) + + def do_POST(self) -> None: # noqa: N802 - http.server API + if self.path.split("?")[0] != "/v1/chat/completions": + self._send(404, {"error": "not found"}) + return + length = int(self.headers.get("Content-Length") or 0) + raw = self.rfile.read(length) if length else b"" + try: + payload = json.loads(raw.decode("utf-8")) + except (ValueError, UnicodeDecodeError): + payload = {} + messages = payload.get("messages") or [] + stub.record( + { + "model": payload.get("model"), + "message_count": len(messages), + "stream": bool(payload.get("stream")), + } + ) + self._send(200, stub.completion_body(str(payload.get("model") or ""))) + + return _Handler diff --git a/e2e/code4me_e2e/suite.py b/e2e/code4me_e2e/suite.py new file mode 100644 index 0000000..b08b040 --- /dev/null +++ b/e2e/code4me_e2e/suite.py @@ -0,0 +1,231 @@ +"""One-command regression gate for backend, plugin, real IDE, website and ACP.""" +from __future__ import annotations + +import contextlib +import io +import json +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from . import browser, native_agents, prereqs, report, stack, ui, workflow +from .steps import STEP_ORDER, StepResult + +#: Layers a full gate attempts, in execution order. ``plugin`` and ``backend`` +#: depend on the IDE layer; ``browser`` is independent and is always attempted, +#: so an IDE failure can never hide a website regression. +ALL_LAYERS: Tuple[str, ...] = ("ui", "plugin", "backend", "agents", "browser") + +#: Layers that are skipped when an earlier planned layer did not pass. +DEPENDENT_LAYERS = frozenset({"plugin", "backend"}) + +#: The report step id each layer records, when it has one. +LAYER_STEP_IDS = {"ui": "ui_test", "plugin": "plugin_test", "agents": "agents_test", "browser": browser.BROWSER_STEP_ID} + + +def plan_layers(layer: str) -> Tuple[str, ...]: + """The layers one invocation attempts, in execution order.""" + if layer == "all": + return ALL_LAYERS + if layer == "plugin": + # ``plugin`` keeps its historical meaning: plugin test plus the backend + # workflow it depends on (the IDE layer is only part of ``all``). + return ("plugin", "backend") + if layer in ("backend", "browser", "agents"): + return (layer,) + raise ValueError(f"unknown layer {layer!r}") + + +@dataclass +class LayerResult: + layer: str + status: str = "NOT_ATTEMPTED" + reason: str = "" + + @property + def attempted(self) -> bool: + return self.status != "NOT_ATTEMPTED" + + def to_dict(self) -> Dict[str, Any]: + return { + "layer": self.layer, + "status": self.status, + "attempted": self.attempted, + "reason": self.reason, + } + + +def _read_report(run_path: Path) -> Dict[str, Any]: + try: + built = json.loads((run_path / "report.json").read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + return built if isinstance(built, dict) else {} + + +def layer_outcome(run_path: Path, *, step_id: Optional[str], rc: int) -> Tuple[str, str]: + """Map one layer's exit code (plus its recorded step) to a gate status. + + A layer that recorded a BLOCKED step is reported as BLOCKED with its + reason; a non-zero exit is never a pass. + """ + built = _read_report(run_path) + if step_id: + step = next( + (item for item in built.get("steps", []) if item.get("id") == step_id), + None, + ) + if step and step.get("status") == "BLOCKED": + details = step.get("details") or {} + reason = str(step.get("fix_hint") or details.get("error") or details.get("message") or "") + return "BLOCKED", reason + if rc == 0 and step and step.get("status") == "PASS": + return "PASS", "" + return "FAIL", "" + # The HTTP workflow layer has no single step: judge it by the workflow's own + # steps only, so another layer's blocked prerequisite cannot relabel it. + statuses = {item.get("status") for item in built.get("steps", []) if item.get("id") in STEP_ORDER} + if rc == 0 and statuses == {"PASS"}: + return "PASS", "" + if "FAIL" in statuses: + return "FAIL", "" + if "BLOCKED" in statuses: + return "BLOCKED", "" + return "FAIL", "" + + +#: Report step recording the prerequisite checks of this gate. +PREREQUISITES_STEP_ID = "prerequisites" + + +def _run_layer(name: str, scenario, run_path: Path, + executables: Optional[Dict[str, str]] = None) -> int: + if name == "ui": + return ui.run_ui_test(scenario, run_dir=str(run_path), keep_stack=True) + if name == "plugin": + print("Testing live Kotlin plugin workflow", file=sys.stderr) + return workflow.run_plugin_test(scenario, run_dir=str(run_path), keep_stack=True) + if name == "backend": + print("Testing HTTP inference, telemetry and enrollment revocation", file=sys.stderr) + return workflow.run_workflow(scenario, from_step="bootstrap", run_dir=str(run_path), keep_stack=True) + if name == "browser": + return browser.run(scenario, run_path) + if name == "agents": + return native_agents.run(scenario, run_path, executables=executables) + raise ValueError(f"unknown layer {name!r}") + + +def _final_report(run_path: Path, scenario, layers: Sequence[LayerResult], + prerequisites: Optional[prereqs.Prerequisites] = None) -> Dict[str, Any]: + path = run_path / "report.json" + if path.is_file(): + try: + built = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + built = None + else: + built = None + if not isinstance(built, dict): + built = report.build_report( + run_id=run_path.name, + started_at=report.utc_now_iso(), + finished_at=report.utc_now_iso(), + scenario=scenario, + steps=[], + findings=[], + ) + built["report_path"] = str(path) + built["layers"] = [item.to_dict() for item in layers] + if prerequisites is not None: + built["prerequisites"] = prerequisites.to_dict() + return built + + +def _record_prerequisites(run_path: Path, scenario, prerequisites: prereqs.Prerequisites, + duration_ms: int) -> None: + """One report step for the host checks; BLOCKED names every unusable one.""" + unusable = [check.to_dict() for check in prerequisites.checks.values() if not check.usable] + status = "BLOCKED" if unusable else "PASS" + fix_hint = "; ".join(f"{item['id']}: {item.get('remediation') or item['detail']}" for item in unusable) + workflow._record_plugin_step(run_path, scenario, StepResult( + PREREQUISITES_STEP_ID, status, duration_ms, + {"checks": [check.to_dict() for check in prerequisites.checks.values()]}, fix_hint, + )) + + +def run(scenario, *, run_dir=None, keep_stack=False, as_json=False, layer="all", + provision: bool = True) -> int: + """Run every planned layer; missing prerequisites are provisioned first. + + With ``provision=False`` nothing is installed or started and a missing + prerequisite blocks its layers exactly like one that cannot be provided. + """ + run_path = Path(run_dir).resolve() if run_dir else report.new_run_dir(workflow.E2E_DIR) + run_path.mkdir(parents=True, exist_ok=True) + layers: Dict[str, LayerResult] = {name: LayerResult(name) for name in plan_layers(layer)} + prerequisites: Optional[prereqs.Prerequisites] = None + rc = 1 + started = time.monotonic() + try: + print(f"E2E artifacts: {run_path}", file=sys.stderr) + print("Checking prerequisites" + (" (provisioning what is missing)" if provision else ""), + file=sys.stderr) + prerequisites = prereqs.prepare(scenario, tuple(layers), provision=provision) + prereqs.print_table(prerequisites) + _record_prerequisites(run_path, scenario, prerequisites, int((time.monotonic() - started) * 1000)) + with contextlib.redirect_stdout(io.StringIO()): + prior: List[LayerResult] = [] + for name in layers: + result = layers[name] + if name in DEPENDENT_LAYERS and any(item.status != "PASS" for item in prior): + result.reason = "not attempted: an earlier dependent layer did not pass" + prior.append(result) + continue + blocked = prerequisites.blocked_reason(name) + if blocked: + # A missing prerequisite is reported, never skipped silently. + result.status, result.reason = "BLOCKED", blocked + prior.append(result) + continue + try: + if layer == "backend": + layer_rc = workflow.run_workflow(scenario, run_dir=str(run_path), keep_stack=True) + else: + layer_rc = _run_layer(name, scenario, run_path, prerequisites.executables) + except Exception as error: + result.status = "FAIL" + result.reason = str(error) + workflow._record_plugin_step(run_path, scenario, StepResult( + "suite", "FAIL", int((time.monotonic() - started) * 1000), + {"layer": name, "error": str(error)}, f"Inspect {run_path}", + )) + report.capture_backend_logs(scenario, run_path) + prior.append(result) + continue + result.status, result.reason = layer_outcome( + run_path, step_id=LAYER_STEP_IDS.get(name), rc=layer_rc + ) + prior.append(result) + rc = 0 if all(item.status == "PASS" for item in layers.values()) else 1 + print("Layers: " + ", ".join(f"{item.layer}={item.status}" for item in layers.values()), file=sys.stderr) + except (Exception, KeyboardInterrupt) as error: + rc = 1 + result = StepResult("suite", "FAIL", int((time.monotonic() - started) * 1000), + {"error": str(error) or "interrupted", "attempted_layers": [item.to_dict() for item in layers.values()]}, + f"Inspect {run_path}") + workflow._record_plugin_step(run_path, scenario, result) + report.capture_backend_logs(scenario, run_path) + finally: + if rc == 0 and not keep_stack and layer != "agents": + try: + stack.down(scenario) + except Exception as error: + rc = 1 + workflow._record_plugin_step(run_path, scenario, + StepResult("cleanup", "FAIL", 0, {"error": str(error)}, "Run stack down")) + built = _final_report(run_path, scenario, list(layers.values()), prerequisites) + report.write_report(run_path, built) + report.print_summary(built, as_json) + return rc diff --git a/e2e/code4me_e2e/ui.py b/e2e/code4me_e2e/ui.py new file mode 100644 index 0000000..48b9d49 --- /dev/null +++ b/e2e/code4me_e2e/ui.py @@ -0,0 +1,313 @@ +"""Launch an isolated real IDE, drive its UI, then act as its ACP host.""" +from __future__ import annotations + +import contextlib +import io +import json +import os +import signal +import socket +import subprocess +import time +import urllib.request +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from . import acp, process, report, runtime, stack +from .config import Scenario +from .paths import E2E_DIR, require_workspace +from .steps import Ctx, StepResult +from .workflow import PLUGIN_PREFIX_STEPS, _plugin_state_usable, _state_still_live, restore_accounts, run_workflow + +UI_TEST_STEP_ID = "ui_test" +UI_TEST_TASK = ":ui-tests:test" +UI_TEST_CLASS = "ui.Code4MeUiNavigationTest" +EXPECTED_UI_STEPS = {"plugin_loaded", "settings_navigation", "sign_in", "enrollment_activation", + "status_surface", "acp_registration", "prepare_agent", "model_response"} + +def parse_results_json(path: Path) -> Dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + if not isinstance(payload, dict): + return {} + return payload + + +def _overall_from_results(results: Dict[str, Any]) -> Tuple[str, List[str]]: + steps = results.get("steps") or [] + reasons: List[str] = [] + statuses = [str(step.get("status", "")).upper() for step in steps] + for step in steps: + status = str(step.get("status", "")).upper() + if status in ("FAIL", "BLOCKED"): + reasons.append(f"{step.get('id')}: {status} - {step.get('reason') or 'no reason recorded'}") + if "FAIL" in statuses: + return "FAIL", reasons + if "BLOCKED" in statuses: + return "BLOCKED", reasons + if statuses and all(status == "PASS" for status in statuses): + return "PASS", reasons + return "UNKNOWN", reasons + + +def parse_gradle_xml(xml_path: Path) -> Dict[str, Any]: + """Minimal JUnit XML summary (tests/failures/errors/skipped + failure text).""" + if not xml_path.is_file(): + return {} + try: + import xml.etree.ElementTree as ET + + root = ET.parse(xml_path).getroot() + except Exception: # noqa: BLE001 - malformed output is reported as unknown + return {} + summary = { + "tests": int(root.get("tests", "0") or 0), + "failures": int(root.get("failures", "0") or 0), + "errors": int(root.get("errors", "0") or 0), + "skipped": int(root.get("skipped", "0") or 0), + "failure_messages": [ + (failure.get("message") or failure.text or "").strip() + for failure in root.iter("failure") + ] + + [ + (error.get("message") or error.text or "").strip() + for error in root.iter("error") + ], + } + return summary + + +def require_test_evidence(exit_code: int, xml: dict, results: dict | None = None, *, log_path: Path | None = None) -> None: + if exit_code or not xml or xml.get("tests", 0) < 1 or any( + xml.get(key, 0) for key in ("failures", "errors", "skipped") + ): + detail = "" + if log_path is not None and log_path.is_file(): + # Surface the first concrete Gradle error instead of a generic hint. + for line in log_path.read_text(errors="ignore").splitlines(): + if line.startswith(("> ", "FAILURE:", "Caused by:")) or "NoSuchFileException" in line: + detail = f": {line.strip()[:200]}" + break + raise RuntimeError( + "Gradle must execute passing tests with no skips; inspect the test log and JUnit XML" + + detail + ) + if results is not None: + steps = results.get("steps") or [] + if {step.get("id") for step in steps} != EXPECTED_UI_STEPS or len(steps) != len(EXPECTED_UI_STEPS): + raise RuntimeError("UI results are missing required steps") + overall, reasons = _overall_from_results(results) + if overall != "PASS": + raise RuntimeError("; ".join(reasons) or "UI results did not pass") + + +# --------------------------------------------------------------------------- +# Post-UI registered-entry check +# --------------------------------------------------------------------------- + + +def verify_registered_entry(run_path: Path) -> Dict[str, Any]: + """Post-UI check: the IDE-registered ACP entry is executable and identified. + + A registration that merely exists in ``acp.json`` is not enough (the + Kotlin ``acp_registration``/``prepare_agent`` steps already check the file + and the bridge). This check resolves the exact command the IDE registered, + requires the proxy *and* the wrapped agent binary to be executable files, + records both digests and ``--version`` outputs, verifies the declared + ``--agent-digest``, and requires the one-time spool capability the proxy + needs to deliver this run's telemetry. + """ + try: + entry = acp.entry_identity(run_path) + except RuntimeError as error: + raise RuntimeError(f"registered ACP entry is unusable: {error}") from error + if "CODE4ME_RESEARCH_CAPABILITY" not in entry.get("env_keys", []): + raise RuntimeError( + "registered ACP entry carries no CODE4ME_RESEARCH_CAPABILITY; the proxy " + "could not deliver this run's telemetry" + ) + try: + uuid.UUID(str(entry.get("research_session_id") or "")) + except ValueError as error: + raise RuntimeError( + "registered ACP entry names no valid CODE4ME_RESEARCH_SESSION_ID; its telemetry " + "cannot be attributed to this run's research session" + ) from error + return entry + + +# --------------------------------------------------------------------------- +# Report recording +# --------------------------------------------------------------------------- + + +def record_ui_step(run_path: Path, scenario: Scenario, result: StepResult, extra: Dict[str, Any]) -> Dict[str, Any]: + """Append/replace the ``ui_test`` step in this run's ``report.json``.""" + report_path = run_path / "report.json" + built: Optional[Dict[str, Any]] = None + if report_path.is_file(): + try: + built = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + built = None + if not isinstance(built, dict): + built = report.build_report( + run_id=run_path.name, + started_at=report.utc_now_iso(), + finished_at=report.utc_now_iso(), + scenario=scenario, + steps=[], + findings=[], + ) + steps = [step for step in built.get("steps", []) if step.get("id") != UI_TEST_STEP_ID] + payload = {**result.to_dict(), **extra} + steps.append(payload) + built["steps"] = steps + built["finished_at"] = report.utc_now_iso() + built["failed_step"] = next((step["id"] for step in steps if step["status"] == "FAIL"), None) + built["blocked_by"] = next((step["id"] for step in steps if step["status"] == "BLOCKED"), None) + built["next_actions"] = report.derive_next_actions(built) + report.write_report(run_path, built) + return built + + +def run_ui_test(scenario: Scenario, *, run_dir: Optional[str] = None, + keep_ide: bool = False, keep_stack: bool = False, + as_json: bool = False) -> int: + run_path = Path(run_dir).resolve() if run_dir else report.new_run_dir(E2E_DIR) + run_path.mkdir(parents=True, exist_ok=True) + plugin = require_workspace() / "code4me2" + home = run_path / "ide-home" + # A fresh isolated home each time; keep previous diagnostics when resuming. + if home.exists(): + home.rename(run_path / f"ide-home-previous-{time.time_ns()}") + home.mkdir(mode=0o700) + # Keep credentials in memory: on macOS the sandbox otherwise shares the + # developer's login keychain (and reads their saved Code4Me account). + security_xml = ('' + '\n') + for config in (home / "config", home / "Library/Application Support/JetBrains/IntelliJIdea2026.2"): + (config / "options").mkdir(parents=True, exist_ok=True) + (config / "options/security.xml").write_text(security_xml) + started = time.monotonic() + child = None + stub = None + status = "FAIL" + details: dict = {} + stage = "prepare" + port = 0 + try: + print(f"Preparing real IDE test: {run_path}", file=__import__('sys').stderr) + gradle_args = runtime.prepare(run_path) + prepare_rc = process.run([str(plugin / "gradlew"), ":prepareSandbox_runIdeForUiTests", + *gradle_args, "--console=plain"], cwd=plugin, + log_path=run_path / "ui-prepareSandbox.log") + if prepare_rc: + raise RuntimeError("Sandbox preparation failed; see ui-prepareSandbox.log") + manifest = json.loads((plugin / "build/research-runtime-staging/proxy-manifest.json").read_text()) + bundle = next(p for p in manifest["platforms"] + if p["os"] == scenario.platform.os and p["arch"] == scenario.platform.arch) + if not bundle["self_contained"]: + raise RuntimeError("The staged proxy must be self-contained") + # The plugin bundles no agent: like a participant whose verified cache + # already holds the study's archive, the IDE home gets the exact archive + # the backend imports (which has no public download URL), pinned by sha256. + _, release_archive = runtime.agent_release(run_path) + digest = runtime.seed_agent_cache(home / "system", release_archive) + scenario.agent.artifact_digest = digest + scenario.agent.release_id = "e2e-ui-" + digest[:20] + scenario.agent.profile_name = "e2e-ui-" + digest[:20] + stage = "workflow_prefix" + state = report.read_state(run_path) + if not (_plugin_state_usable(state) and _state_still_live(scenario, state)): + with contextlib.redirect_stdout(io.StringIO()): + rc = run_workflow(scenario, only=",".join(PLUGIN_PREFIX_STEPS), + run_dir=str(run_path), keep_stack=True) + if rc: + raise RuntimeError("Workflow prefix failed; see report.json") + state = report.read_state(run_path) + restore_accounts(scenario, state) + ctx = Ctx(scenario, state, run_path) + ctx.ensure_stub() + stub = ctx.stub + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + robot_url = f"http://127.0.0.1:{port}" + results_file = run_path / "ui-test-results.json" + xml_path = plugin / "ui-tests/build/test-results/test/TEST-ui.Code4MeUiNavigationTest.xml" + results_file.unlink(missing_ok=True) + xml_path.unlink(missing_ok=True) + env = {**os.environ, "CODE4ME_UI_TEST": "1", "CODE4ME_UI_HOME": str(home), + "CODE4ME_E2E_BASE_URL": scenario.base_url, + "CODE4ME_E2E_EMAIL": scenario.participant.email, + "CODE4ME_E2E_PASSWORD": scenario.participant.password, + "CODE4ME_E2E_JOIN_CODE": state["join_code"], + "CODE4ME_UI_ROBOT_URL": robot_url, "CODE4ME_UI_ROBOT_PORT": str(port), + "CODE4ME_UI_PROJECT_DIR": str(run_path / "ui-project"), + "CODE4ME_UI_RESULT_FILE": str(results_file), + "CODE4ME_UI_EXPECTED_RESPONSE": scenario.message.expected_substring or state["stub_token"], + "CODE4ME_UI_MODEL_PROMPT": scenario.message.prompt} + stage = "ide_startup" + with (run_path / "ui-ide.log").open("w") as log: + child = subprocess.Popen([str(plugin / "gradlew"), ":runIdeForUiTests", *gradle_args, + "--no-daemon", "--console=plain"], cwd=plugin, env=env, + stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + deadline = time.monotonic() + int(os.environ.get("CODE4ME_UI_IDE_TIMEOUT_SECONDS", "600")) + while time.monotonic() < deadline: + if child.poll() is not None: + raise RuntimeError("IDE exited before robot-server became ready; see ui-ide.log") + try: + with urllib.request.urlopen(robot_url + "/hello", timeout=2) as response: + if response.status == 200: + break + except OSError: + pass + time.sleep(1) + else: + raise TimeoutError("IDE startup timed out; see ui-ide.log") + stage = "ui_navigation" + print("IDE ready; exercising login, enrollment and agent preparation", file=__import__('sys').stderr) + rc = process.run([str(plugin / "gradlew"), UI_TEST_TASK, "--tests", UI_TEST_CLASS, + "--rerun", "--console=plain"], cwd=plugin, env=env, + log_path=run_path / "ui-test.log", timeout=900) + results = parse_results_json(results_file) + details["per_step"] = results.get("steps", []) + xml = parse_gradle_xml(xml_path) + details["junit"] = xml + require_test_evidence(rc, xml, results, log_path=run_path / 'ui-test.log') + stage = "registered_entry" + # A visible registration can still be unusable: resolve the exact entry + # command/agent argv, digest and version before launching it. + details["registered_entry"] = verify_registered_entry(run_path) + stage = "acp_conversation" + print("UI passed; testing registered ACP proxy, model turn and telemetry", file=__import__('sys').stderr) + details["acp"] = acp.exercise(scenario, run_path, state, stub) + status = "PASS" + except Exception as error: + details.update(stage=stage, error=str(error)) + report.capture_backend_logs(scenario, run_path) + finally: + if not keep_ide and child is not None: + # Gradle may use a detached single-use daemon. Match both this run's + # unique home and robot port, never another sandbox or developer IDE. + listing = subprocess.run(["ps", "-ww", "-eo", "pid=,command="], capture_output=True, text=True).stdout + for line in listing.splitlines(): + if f"-Duser.home={home}" in line and f"-Drobot-server.port={port}" in line: + try: + os.kill(int(line.strip().split()[0]), signal.SIGTERM) + except ProcessLookupError: + pass + process.stop(child) + if stub: + stub.stop() + result = StepResult(UI_TEST_STEP_ID, status, int((time.monotonic() - started) * 1000), + details, "" if status == "PASS" else f"Inspect {run_path}/ui-test.log and ui-ide.log ({stage})") + built = record_ui_step(run_path, scenario, result, {}) + report.print_summary(built, as_json) + if status == "PASS" and not keep_stack and not keep_ide: + stack.down(scenario) + return 0 if status == "PASS" else 1 diff --git a/e2e/code4me_e2e/workflow.py b/e2e/code4me_e2e/workflow.py new file mode 100644 index 0000000..5f9c6e4 --- /dev/null +++ b/e2e/code4me_e2e/workflow.py @@ -0,0 +1,465 @@ +"""Workflow orchestration: step selection, resumable execution, reporting.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import subprocess +import sys +import time +import traceback +from pathlib import Path +from typing import Any, Dict, List, Optional + +from . import report, stack +from .config import Scenario +from .http import HttpError +from .paths import E2E_DIR, require_workspace +from .steps import STEP_ORDER, STEPS, Ctx, StepBlocked, StepFailure, StepResult + +#: Steps that leave an ACTIVE enrollment the plugin test can bootstrap. +PLUGIN_PREFIX_STEPS: List[str] = STEP_ORDER[: STEP_ORDER.index("plugin_join") + 1] + +#: Gradle task + test class the plugin-in-IntelliJ layer runs. +PLUGIN_TEST_TASK = ":integration-tests:test" +PLUGIN_TEST_CLASS = "integration.LiveStudyWorkflowPluginTest" +PLUGIN_TEST_STEP_ID = "plugin_test" + + +class WorkflowError(RuntimeError): + pass + + +def _select_steps(only: Optional[str], from_step: Optional[str]) -> List[str]: + steps = list(STEP_ORDER) + if from_step: + if from_step not in STEP_ORDER: + raise WorkflowError( + f"unknown step {from_step!r}; known steps: {', '.join(STEP_ORDER)}" + ) + steps = steps[steps.index(from_step) :] + if only: + requested = [item.strip() for item in only.split(",") if item.strip()] + for step in requested: + if step not in STEP_ORDER: + raise WorkflowError( + f"unknown step {step!r}; known steps: {', '.join(STEP_ORDER)}" + ) + steps = [step for step in steps if step in requested] + return steps + + +def _own_stack(scenario: Scenario) -> bool: + base = (scenario.base_url or "").rstrip("/") + return base in ( + f"http://localhost:{scenario.stack.backend_port}", + f"http://127.0.0.1:{scenario.stack.backend_port}", + ) + + +def _stack_ready(scenario: Scenario) -> bool: + status, payload = stack.capabilities(scenario) + return status == 200 and isinstance(payload, dict) and bool(payload.get("schema_ready")) + + +def _server_reason(record: Dict[str, Any]) -> str: + """The backend's own reason for a handled server error: its ``detail`` code or text.""" + response = record.get("response") + detail = response.get("detail") if isinstance(response, dict) else None + if isinstance(detail, dict): + detail = detail.get("code") or detail.get("message") + return str(detail)[:200] if detail else "no detail" + + +def _run_one(ctx: Ctx, step_id: str) -> StepResult: + start_index = len(ctx.exchange_log) + started = time.monotonic() + status = "PASS" + details: Dict[str, Any] = {} + fix_hint = "" + try: + details = STEPS[step_id](ctx) or {} + except StepBlocked as error: + status = "BLOCKED" + details = {**error.details, "message": error.message} + fix_hint = error.fix_hint + except StepFailure as error: + status = "FAIL" + details = {**error.details, "message": error.message} + fix_hint = error.fix_hint + except HttpError as error: + status = "FAIL" + details = {"error": str(error)} + fix_hint = "the backend could not be reached; verify the stack is up" + except Exception as error: # noqa: BLE001 - a crash is a failed step, never a lost run + status = "FAIL" + details = {"error": repr(error), "traceback": traceback.format_exc()} + fix_hint = "unexpected harness error; see traceback in report.json" + duration_ms = int((time.monotonic() - started) * 1000) + + exchanges = ctx.records_since(start_index, limit=1000) + if status != "PASS" and exchanges: + details = {**details, "exchanges": exchanges[-6:]} + if status == "FAIL": + server_errors = [ + record + for record in exchanges + if isinstance(record.get("status"), int) and record["status"] >= 500 + ] + if server_errors: + # Only a 500 is an unhandled exception. The backend's deliberate 503s + # (store, provider or configuration unavailable) and gateway 502/504s + # point at the environment. The last error is cited: an earlier one may + # have been retried. The two kinds keep separate ids, so a resumed run + # that now crashes still reports the bug. + crashes = [record for record in server_errors if record["status"] == 500] + error = (crashes or server_errors)[-1] + where = f"HTTP {error['status']} from {error['method']} {error['path']}" + finding_id = f"BACKEND_{'500' if crashes else '5XX'}_{step_id.upper()}" + if not any(item.get("id") == finding_id for item in ctx.findings): + if crashes: + finding = { + "severity": "bug", + "message": ( + f"step {step_id} received {where}: an unhandled backend exception. " + "This is an application bug, not a harness failure; the harness " + "keeps the exact request/response and the backend traceback for " + "the report." + ), + "location": "see logs/backend.log for the Python traceback", + } + else: + finding = { + "severity": "environment", + "message": ( + f"step {step_id} received {where} ({_server_reason(error)}): the " + "backend or a gateway reported something it depends on as " + "unavailable. Check the stack, the provider stub and the backend " + "configuration before suspecting the application." + ), + "location": "see the step's exchanges in report.json and logs/backend.log", + } + ctx.findings.append({"id": finding_id, **finding}) + ctx.state["findings"] = ctx.findings + return StepResult(step_id, status, duration_ms, details, fix_hint) + + +def run_workflow( + scenario: Scenario, + *, + only: Optional[str] = None, + from_step: Optional[str] = None, + run_dir: Optional[str] = None, + keep_stack: bool = False, + as_json: bool = False, +) -> int: + steps = _select_steps(only, from_step) + if not steps: + raise WorkflowError("no steps selected") + + if run_dir: + run_path = Path(run_dir).resolve() + elif from_step: + run_path = report.latest_run_dir(E2E_DIR) or report.new_run_dir(E2E_DIR) + else: + run_path = report.new_run_dir(E2E_DIR) + run_path.mkdir(parents=True, exist_ok=True) + + state = report.read_state(run_path) + # Fresh accounts make repetitions independent of terminal enrollments and + # immutable release/profile selections. Resume with the same credentials. + if not state: + from dataclasses import asdict + for role in ("admin", "researcher", "participant"): + account = getattr(scenario, role) + local, domain = account.email.rsplit("@", 1) + # The full path: layer state directories share names (browser-state). + suffix = __import__("hashlib").sha256(str(run_path).encode()).hexdigest()[:10] + account.email = f"{local[:38-len(domain)]}+{suffix}@{domain}" + state["accounts"] = {role: asdict(getattr(scenario, role)) + for role in ("admin", "researcher", "participant")} + else: + restore_accounts(scenario, state) + started_at = report.utc_now_iso() + run_id = run_path.name + + up_started = False + if _own_stack(scenario): + up_started = not _stack_ready(scenario) + stack.up(scenario) + + ctx = Ctx(scenario, state, run_path, fresh_login="create_accounts" not in steps) + ctx.findings = list(state.get("findings") or []) + + results: List[Dict[str, Any]] = [] + failed_step: Optional[str] = None + blocked_by: Optional[str] = None + + for index, step_id in enumerate(steps): + start_index = len(ctx.exchange_log) + result = _run_one(ctx, step_id) + report.append_http(run_path, ctx.exchange_log[start_index:]) + results.append(result.to_dict()) + state.setdefault("step_results", {})[step_id] = result.to_dict() + ctx.snapshot_cookies() + state["findings"] = ctx.findings + report.write_state(run_path, state) + + if result.status in ("FAIL", "BLOCKED"): + if result.status == "BLOCKED": + blocked_by = step_id + else: + failed_step = step_id + for remaining in steps[index + 1 :]: + results.append( + StepResult( + remaining, + "SKIP", + 0, + {"reason": f"not run after {step_id}"}, + "", + ).to_dict() + ) + break + + finished_at = report.utc_now_iso() + # This invocation passes or fails on the steps it ran. Steps merged from the + # run's shared report (another layer's result, the gate's prerequisites + # record) stay in the report but must not decide this return code. + own_ok = not any(item["status"] in ("FAIL", "BLOCKED") for item in results) + previous = run_path / "report.json" + if previous.is_file(): + old = json.loads(previous.read_text()).get("steps", []) + selected = {item["id"] for item in results} + results = [item for item in old if item["id"] not in selected] + results + failed_step = next((s["id"] for s in results if s["status"] == "FAIL"), None) + blocked_by = next((s["id"] for s in results if s["status"] == "BLOCKED"), None) + built = report.build_report( + run_id=run_id, + started_at=started_at, + finished_at=finished_at, + scenario=scenario, + steps=results, + findings=ctx.findings, + failed_step=failed_step, + blocked_by=blocked_by, + ) + report.write_report(run_path, built) + + if not own_ok: + log_path = report.capture_backend_logs(scenario, run_path) + built["backend_log"] = log_path + report.write_report(run_path, built) + + report.print_summary(built, as_json) + + if own_ok and up_started and not keep_stack: + try: + stack.down(scenario) + except Exception as error: # noqa: BLE001 + print(f"warning: could not tear the stack down: {error}", file=sys.stderr) + + if ctx.stub: + ctx.stub.stop() + return 0 if own_ok else 1 + + +def run_single_step( + scenario: Scenario, + step_id: str, + *, + run_dir: Optional[str] = None, + as_json: bool = False, +) -> int: + if step_id not in STEP_ORDER: + raise WorkflowError( + f"unknown step {step_id!r}; known steps: {', '.join(STEP_ORDER)}" + ) + if run_dir: + run_path = Path(run_dir).resolve() + else: + run_path = report.latest_run_dir(E2E_DIR) or report.new_run_dir(E2E_DIR) + run_path.mkdir(parents=True, exist_ok=True) + state = report.read_state(run_path) + restore_accounts(scenario, state) + ctx = Ctx(scenario, state, run_path, fresh_login=step_id != "create_accounts") + ctx.findings = list(state.get("findings") or []) + result = _run_one(ctx, step_id) + state.setdefault("step_results", {})[step_id] = result.to_dict() + ctx.snapshot_cookies() + state["findings"] = ctx.findings + report.write_state(run_path, state) + report.append_http(run_path, ctx.exchange_log) + if as_json: + print(json.dumps(result.to_dict(), indent=2, default=str)) + else: + print(f"{result.id} {result.status} ({result.duration_ms} ms)") + if result.details: + print(json.dumps(result.details, indent=2, default=str)) + if result.fix_hint: + print(f"fix: {result.fix_hint}") + if ctx.stub: + ctx.stub.stop() + return 0 if result.status == "PASS" else 1 + + +def run_doctor(scenario: Scenario, *, as_json: bool = False) -> int: + """Run only the ``doctor`` step, without creating a run directory.""" + ctx = Ctx(scenario, {}, report.runs_root(E2E_DIR)) + result = _run_one(ctx, "doctor") + if as_json: + print(json.dumps(result.to_dict(), indent=2, default=str)) + else: + print(f"doctor {result.status} ({result.duration_ms} ms)") + print(json.dumps(result.details, indent=2, default=str)) + if result.fix_hint: + print(f"fix: {result.fix_hint}") + if ctx.stub: + ctx.stub.stop() + return 0 if result.status == "PASS" else 1 + + +# --------------------------------------------------------------------------- +# Plugin layer: run the plugin's own Kotlin code inside an IntelliJ fixture +# against the live backend. +# --------------------------------------------------------------------------- + + +def restore_accounts(scenario: Scenario, state: dict) -> None: + for role, values in state.get("accounts", {}).items(): + if role in ("admin", "researcher", "participant"): + for key in ("email", "password", "name"): + setattr(getattr(scenario, role), key, values[key]) + + +def _plugin_state_usable(state: Dict[str, Any]) -> bool: + """True when ``state`` leaves an ACTIVE enrollment for the plugin test. + + Requires a join code, an enrollment id, a PASSING ``plugin_join``, and no + completed ``revoke`` (which revokes the enrollment). + """ + if not state: + return False + if not state.get("join_code") or not state.get("enrollment_id"): + return False + results = state.get("step_results") or {} + if (results.get("plugin_join") or {}).get("status") != "PASS": + return False + if (results.get("revoke") or {}).get("status") == "PASS": + return False + return True + + +def _find_usable_run_dir(scenario: Scenario, e2e_dir: Path) -> Optional[Path]: + """Newest existing run whose state still leaves an ACTIVE enrollment.""" + root = report.runs_root(e2e_dir) + if not root.is_dir(): + return None + candidates = sorted( + (path for path in root.iterdir() if (path / "state.json").is_file()), + key=lambda path: path.stat().st_mtime, + reverse=True, + ) + for path in candidates: + state = report.read_state(path) + if _plugin_state_usable(state) and _state_still_live(scenario, state): + return path + return None + + +def _state_still_live(scenario: Scenario, state: Dict[str, Any]) -> bool: + """True when the state's enrollment is still ACTIVE on the current stack. + + A run recorded before ``stack down -v`` points at a database that no longer + exists; reusing it would send the plugin at a dead enrollment. For a stack + the harness does not own the database cannot be probed, so the structural + state is trusted. + """ + enrollment_id = state.get("enrollment_id") + if not enrollment_id: + return False + if not _own_stack(scenario): + return True + ok, out = stack.try_psql( + scenario, + "SELECT status FROM research_enrollment WHERE enrollment_id='" + + str(enrollment_id).replace("'", "") + + "';", + ) + return ok and out.strip().upper() == "ACTIVE" + + +def _record_plugin_step(run_path: Path, scenario: Scenario, result: StepResult) -> Dict[str, Any]: + """Append/replace the ``plugin_test`` step in this run's ``report.json``.""" + report_path = run_path / "report.json" + if report_path.is_file(): + try: + built = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + built = None + else: + built = None + if not isinstance(built, dict): + built = report.build_report( + run_id=run_path.name, + started_at=report.utc_now_iso(), + finished_at=report.utc_now_iso(), + scenario=scenario, + steps=[], + findings=[], + ) + steps = [step for step in built.get("steps", []) if step.get("id") != result.id] + steps.append(result.to_dict()) + built["steps"] = steps + built["finished_at"] = report.utc_now_iso() + built["failed_step"] = next((step["id"] for step in steps if step["status"] == "FAIL"), None) + built["blocked_by"] = next((step["id"] for step in steps if step["status"] == "BLOCKED"), None) + built["next_actions"] = report.derive_next_actions(built) + report.write_report(run_path, built) + return built + + +def run_plugin_test(scenario: Scenario, *, run_dir: Optional[str] = None, + keep_stack: bool = False, as_json: bool = False) -> int: + from . import process + from .ui import parse_gradle_xml, require_test_evidence + plugin = require_workspace() / "code4me2" + run_path = Path(run_dir).resolve() if run_dir else report.new_run_dir(E2E_DIR) + run_path.mkdir(parents=True, exist_ok=True) + started = time.monotonic() + details = {} + status = "FAIL" + try: + state = report.read_state(run_path) + if not (_plugin_state_usable(state) and _state_still_live(scenario, state)): + with contextlib.redirect_stdout(io.StringIO()): + rc = run_workflow(scenario, only=",".join(PLUGIN_PREFIX_STEPS), + run_dir=str(run_path), keep_stack=True) + if rc: + raise RuntimeError("Workflow prefix failed; see report.json") + state = report.read_state(run_path) + restore_accounts(scenario, state) + env = {**os.environ, "CODE4ME_E2E_BASE_URL": scenario.base_url, + "CODE4ME_E2E_EMAIL": scenario.participant.email, + "CODE4ME_E2E_PASSWORD": scenario.participant.password, + "CODE4ME_E2E_JOIN_CODE": state["join_code"]} + xml_path = plugin / "integration-tests/build/test-results/test/TEST-integration.LiveStudyWorkflowPluginTest.xml" + xml_path.unlink(missing_ok=True) + rc = process.run([str(plugin / "gradlew"), PLUGIN_TEST_TASK, "--tests", PLUGIN_TEST_CLASS, + "--rerun", "--console=plain"], cwd=plugin, env=env, + log_path=run_path / "plugin-test.log", timeout=600) + details = {"exit_code": rc, "junit": parse_gradle_xml(xml_path)} + require_test_evidence(rc, details["junit"]) + status = "PASS" + except Exception as error: + details["error"] = str(error) + report.capture_backend_logs(scenario, run_path) + result = StepResult(PLUGIN_TEST_STEP_ID, status, int((time.monotonic()-started)*1000), details, + "" if status == "PASS" else f"Inspect {run_path}/plugin-test.log") + built = _record_plugin_step(run_path, scenario, result) + report.print_summary(built, as_json) + if status == "PASS" and not keep_stack: + stack.down(scenario) + return 0 if status == "PASS" else 1 diff --git a/e2e/docker-compose.e2e.yml b/e2e/docker-compose.e2e.yml new file mode 100644 index 0000000..0b114b2 --- /dev/null +++ b/e2e/docker-compose.e2e.yml @@ -0,0 +1,104 @@ +# Disposable, isolated backend stack for the code4me-e2e harness. +# +# This stack is intentionally separate from the developer stack: it always runs +# under its own compose project name and its own named volume, uses distinct +# host ports, and never sets a fixed container_name. It never stops, restarts or +# removes the developer containers/volumes. +# +# Bring it up with the harness, not by hand: +# python3 -m code4me_e2e stack up +# +# Interpolation variables all carry a safe default so that +# `docker compose -f e2e/docker-compose.e2e.yml config` is valid with no env. +# The harness always passes real (non-default) values for ports and secrets. + +services: + db: + image: pgvector/pgvector:pg16 + environment: + POSTGRES_USER: ${E2E_DB_USER:-postgres} + POSTGRES_PASSWORD: ${E2E_DB_PASSWORD:-postgres} + POSTGRES_DB: ${E2E_DB_NAME:-code4me_e2e} + volumes: + - code4me_e2e_dbdata:/var/lib/postgresql/data + - ../src/database/init.sql:/docker-entrypoint-initdb.d/init.sql:ro + ports: + - "127.0.0.1:${E2E_DB_PORT:-25432}:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 3s + timeout: 5s + retries: 40 + + redis: + image: redis:7.0 + ports: + - "127.0.0.1:${E2E_REDIS_PORT:-26379}:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 5s + retries: 40 + + backend: + image: ${E2E_BACKEND_IMAGE:-code4me2-server-backend:latest} + working_dir: /app + volumes: + - ../src:/app/src:ro + - ../alembic.ini:/app/alembic.ini:ro + - ./backend.env:/app/.env:ro + env_file: + - ./backend.env + environment: + # Compose `environment` wins over `env_file`, so the disposable stack's + # connection targets never leak in from the developer `.env`. + DB_HOST: db + DB_PORT: "5432" + DB_NAME: ${E2E_DB_NAME:-code4me_e2e} + DB_USER: ${E2E_DB_USER:-postgres} + DB_PASSWORD: ${E2E_DB_PASSWORD:-postgres} + REDIS_HOST: redis + REDIS_PORT: "6379" + CELERY_BROKER_HOST: redis + CELERY_BROKER_PORT: "6379" + SERVER_HOST: 0.0.0.0 + SERVER_PORT: "8008" + PYTHONPATH: /app/src + PRELOAD_MODELS: "false" + MODEL_USE_CACHE: "false" + MODEL_WARMUP: "false" + MODEL_USE_COMPILE: "false" + BOOTSTRAP_SIGNING_SECRET: ${E2E_BOOTSTRAP_SIGNING_SECRET:-e2e-bootstrap-secret} + E2E_STUB_API_KEY: ${E2E_STUB_API_KEY:-e2e-stub-secret-value} + # Read by the provider-backed classic rows (seeded rows 1 and 3); empty + # unless the caller exports it (tests/test_classic_provider_e2e.py does). + OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-} + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "127.0.0.1:${E2E_BACKEND_PORT:-28008}:8008" + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + # Fresh disposable DB: init.sql runs on first volume creation, then the + # migrations bring it to head (adding can_research/is_research/provider_*). + command: + - /bin/bash + - -c + - source activate myenv && alembic upgrade head && exec python src/main.py + healthcheck: + test: + - CMD-SHELL + - >- + /opt/conda/envs/myenv/bin/python -c "import urllib.request as u; + r=u.urlopen(u.Request('http://localhost:8008/api/ping', method='HEAD'), timeout=5); + exit(0 if r.status == 200 else 1)" + interval: 5s + timeout: 10s + retries: 60 + start_period: 30s + +volumes: + code4me_e2e_dbdata: diff --git a/e2e/scenario.example.json b/e2e/scenario.example.json new file mode 100644 index 0000000..d67bf14 --- /dev/null +++ b/e2e/scenario.example.json @@ -0,0 +1,89 @@ +{ + "_comment": "Example scenario for `python3 -m code4me_e2e run --scenario scenario.example.json`. Every key is optional; the harness has the same defaults built in. Keys beginning with `_` or `$` are documentation only. Override any value from the CLI with `--set dotted.key=value`.", + "base_url": "http://localhost:28008", + + "stack": { + "_comment": "The harness's own disposable docker-compose stack.", + "project_name": "code4me-e2e", + "backend_port": 28008, + "db_port": 25432, + "redis_port": 26379, + "image": "code4me2-server-backend:latest", + "bootstrap_signing_secret": "e2e-bootstrap-secret", + "stub_secret_env": "E2E_STUB_API_KEY", + "stub_secret_value": "e2e-stub-secret-value", + "db_password": "postgres", + "db_name": "code4me_e2e", + "db_user": "postgres", + "stub_port": 28999 + }, + + "admin": { + "_comment": "A fresh admin account the harness creates; there is no admin endpoint, so `grant_roles` promotes it via SQL. Emails must use a normal TLD (`.test` is rejected by EmailStr).", + "email": "e2e-admin@example.com", + "password": "AdminPass123", + "name": "E2E Admin" + }, + + "researcher": { + "email": "e2e-researcher@example.com", + "password": "ResearcherPass123", + "name": "E2E Researcher" + }, + + "participant": { + "email": "e2e-participant@example.com", + "password": "ParticipantPass123", + "name": "E2E Participant" + }, + + "study": { + "name": "E2E Synthetic Study", + "description": "Synthetic study created by the code4me-e2e harness.", + "owner": "e2e-research-ops", + "start_at": null, + "end_at": null + }, + + "agent": { + "model": "e2e-stub-model", + "framework_version": "code4me2-agent", + "approval_policy": "auto", + "max_steps": 4, + "agent_id": "code4me2-e2e-agent", + "release_id": "e2e-release-1", + "release_version": "1.0.0", + "artifact_digest": null, + "artifact_path": null, + "artifact_size": 1024, + "profile_name": "e2e-profile", + "_probe_comment": "Inputs of the standalone agent-probe only (--layer agents runs both frameworks and uses discovery plus CODE4ME_E2E__EXECUTABLE). executable replaces discovery; agent_command/agent_package name the agent like a BYOA release; agent_command_args replaces the default ACP argv; agent_home replaces the isolated per-run home (must be empty).", + "agent_command": null, + "agent_command_args": [], + "agent_package": null, + "executable": null, + "agent_home": null, + "adapter_id": null, + "adapter_version": "1.0.0", + "adapter_digest": null + }, + + "platform": { + "os": "macos", + "arch": "aarch64", + "ide_build": "IU-243.1", + "host_kind": "IntelliJ IDEA", + "plugin_version": "e2e" + }, + + "message": { + "_comment": "expected_substring null means: assert the stub provider's own deterministic token.", + "prompt": "Say hello in one short sentence.", + "expected_substring": null + }, + + "timeouts": { + "backend_ready_seconds": 300, + "step_seconds": 120 + } +} diff --git a/e2e/test b/e2e/test new file mode 100755 index 0000000..1b072be --- /dev/null +++ b/e2e/test @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +"""Run from any directory: ./e2e/test [--layer all|backend|plugin|agents|browser] [--json] [--no-setup]. + +Checks and provisions each layer's prerequisites first (see README "For coding agents"). +""" +from code4me_e2e.cli import main +import sys + +raise SystemExit(main(["test", *sys.argv[1:]])) diff --git a/e2e/tests/_support.py b/e2e/tests/_support.py new file mode 100644 index 0000000..605e1a5 --- /dev/null +++ b/e2e/tests/_support.py @@ -0,0 +1,22 @@ +"""Shared unit-test isolation: suite tests must never touch the real host.""" +from __future__ import annotations + +from unittest.mock import patch + +from code4me_e2e import prereqs, suite + + +def ready_prerequisites(scenario, layers, *, provision=True): + """Every prerequisite of ``layers`` READY, without running a single check.""" + result = prereqs.Prerequisites(tuple(layers)) + for layer in layers: + for name in prereqs.LAYER_REQUIREMENTS.get(layer, ()): + result.checks.setdefault(name, prereqs.Check(name, prereqs.READY, "unit-test fake")) + return result + + +def isolate_prerequisites(test_case) -> None: + """Patch the gate's host checks for the duration of ``test_case``.""" + patcher = patch.object(suite.prereqs, "prepare", side_effect=ready_prerequisites) + patcher.start() + test_case.addCleanup(patcher.stop) diff --git a/e2e/tests/test_agent_provider.py b/e2e/tests/test_agent_provider.py new file mode 100644 index 0000000..daf9aed --- /dev/null +++ b/e2e/tests/test_agent_provider.py @@ -0,0 +1,146 @@ +"""HTTP-level contract tests for the native-agent loopback provider.""" + +from __future__ import annotations + +import json +import unittest +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +from code4me_e2e.agent_provider import AgentProvider, NativeAgentProvider + + +class AgentProviderTest(unittest.TestCase): + def setUp(self) -> None: + self.provider = AgentProvider(token="E2E_NATIVE_ANSWER_TEST_MARKER") + self.provider.start() + + def tearDown(self) -> None: + self.provider.stop() + + def request(self, route: str, body: object) -> tuple[int, str, str]: + request = Request( + self.provider.base_url + route, + data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json", "Authorization": "Bearer never-record-me"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: # noqa: S310 - loopback fixture + return response.status, response.headers.get_content_type(), response.read().decode("utf-8") + + def test_chat_completion_json_and_metadata_only_receipt(self) -> None: + status, content_type, body = self.request("/v1/chat/completions", { + "model": "goose-model", "stream": False, "messages": [{"role": "user", "content": "private prompt"}], + }) + self.assertEqual((200, "application/json"), (status, content_type)) + self.assertEqual("E2E_NATIVE_ANSWER_TEST_MARKER", json.loads(body)["choices"][0]["message"]["content"]) + self.assertEqual([{ + "route": "/v1/chat/completions", "model": "goose-model", "stream": False, + "count": 1, "receipt": 1, "auth_matched": None, + }], self.provider.requests_since(0)) + self.assertNotIn("private prompt", repr(self.provider.requests)) + self.assertNotIn("never-record-me", repr(self.provider.requests)) + + def test_chat_completion_sse_ends_with_done(self) -> None: + status, content_type, body = self.request("/v1/chat/completions", {"model": "goose-stream", "stream": True}) + self.assertEqual((200, "text/event-stream"), (status, content_type)) + self.assertIn('"content":"E2E_NATIVE_ANSWER_TEST_MARKER"', body) + self.assertTrue(body.endswith("data: [DONE]\n\n")) + self.assertEqual("/v1/chat/completions", self.provider.requests_since(0)[0]["route"]) + self.assertTrue(self.provider.requests_since(0)[0]["stream"]) + + def test_responses_json_and_sse_are_complete(self) -> None: + status, content_type, body = self.request("/v1/responses", {"model": "codex-model", "stream": False}) + self.assertEqual((200, "application/json"), (status, content_type)) + self.assertEqual("E2E_NATIVE_ANSWER_TEST_MARKER", json.loads(body)["output"][0]["content"][0]["text"]) + + status, content_type, body = self.request("/v1/responses", {"model": "codex-stream", "stream": True}) + self.assertEqual((200, "text/event-stream"), (status, content_type)) + self.assertIn("event: response.output_text.delta", body) + self.assertIn('"type":"response.completed"', body) + receipts = self.provider.requests_since(0) + self.assertEqual(["/v1/responses", "/v1/responses"], [item["route"] for item in receipts]) + self.assertEqual([False, True], [item["stream"] for item in receipts]) + + def test_unsupported_and_invalid_requests_fail_without_receipts(self) -> None: + with self.assertRaises(HTTPError) as unsupported: + self.request("/v1/embeddings", {"model": "ignored"}) + self.assertEqual(404, unsupported.exception.code) + self.assertEqual(0, self.provider.request_count()) + + malformed = Request(self.provider.base_url + "/v1/responses", data=b"{", method="POST") + with self.assertRaises(HTTPError) as invalid: + urlopen(malformed, timeout=5) # noqa: S310 - loopback fixture + self.assertEqual(400, invalid.exception.code) + self.assertEqual(0, self.provider.request_count()) + + def test_lifecycle_is_idempotent_and_alias_is_compatible(self) -> None: + self.assertIs(NativeAgentProvider, AgentProvider) + port = self.provider.port + self.assertEqual(port, self.provider.start()) + self.provider.stop() + self.provider.stop() + + +if __name__ == "__main__": + unittest.main() + + +class GatewayShapeProviderTest(unittest.TestCase): + """The research inference gateway route, bearer receipts and the 402 refusal.""" + + GATEWAY = "/api/research/inference/v1/chat/completions" + + def request(self, provider, route, body, *, bearer): + request = Request( + provider.base_url + route, + data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json", "Authorization": f"Bearer {bearer}"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: # noqa: S310 - loopback fixture + return response.status, json.loads(response.read().decode("utf-8")) + + def test_gateway_route_answers_and_records_only_whether_the_bearer_matched(self): + provider = AgentProvider(token="E2E_GATEWAY_MARKER", expected_bearer="c4m-secret-bearer") + provider.start() + try: + status, body = self.request(provider, self.GATEWAY, {"model": "m", "stream": False}, bearer="c4m-secret-bearer") + self.assertEqual(200, status) + self.assertEqual("E2E_GATEWAY_MARKER", body["choices"][0]["message"]["content"]) + status, _ = self.request(provider, self.GATEWAY, {"model": "m", "stream": False}, bearer="wrong") + self.assertEqual(200, status) + receipts = provider.requests_since(0) + self.assertEqual([True, False], [item["auth_matched"] for item in receipts]) + self.assertEqual([self.GATEWAY, self.GATEWAY], [item["route"] for item in receipts]) + self.assertNotIn("c4m-secret-bearer", repr(receipts)) + finally: + provider.stop() + + def test_quota_exhausted_mode_refuses_with_the_gateway_body(self): + provider = AgentProvider(quota_exhausted=True) + provider.start() + try: + with self.assertRaises(HTTPError) as refused: + self.request(provider, self.GATEWAY, {"model": "m", "stream": True}, bearer="x") + self.assertEqual(402, refused.exception.code) + body = json.loads(refused.exception.read().decode("utf-8")) + self.assertEqual("insufficient_quota", body["error"]["type"]) + self.assertEqual("quota_exhausted", body["error"]["code"]) + self.assertIn("budget is used up", body["error"]["message"]) + self.assertEqual(1, provider.request_count()) + # A Responses call is unaffected by the chat refusal mode. + status, _ = self.request(provider, "/v1/responses", {"model": "m", "stream": False}, bearer="x") + self.assertEqual(200, status) + finally: + provider.stop() + + def test_models_listing_stays_unsupported(self): + provider = AgentProvider() + provider.start() + try: + with self.assertRaises(HTTPError) as unsupported: + urlopen(Request(provider.base_url + "/v1/models", method="GET"), timeout=5) # noqa: S310 + self.assertEqual(404, unsupported.exception.code) + finally: + provider.stop() diff --git a/e2e/tests/test_classic_provider_e2e.py b/e2e/tests/test_classic_provider_e2e.py new file mode 100644 index 0000000..8b752d2 --- /dev/null +++ b/e2e/tests/test_classic_provider_e2e.py @@ -0,0 +1,154 @@ +"""Live end-to-end check: classic completion and chat answered by a provider. + +A freshly seeded database must serve the plugin's default classic rows (id 1 +completion, id 3 chat) through OpenRouter, while the other rows stay local and are +only loaded when explicitly chosen. The test runs on its own disposable stack +(project ``code4me-e2e-classic``), never the developer stack, and makes a few real, +cheap OpenRouter calls, so it is opt-in: + + cd code4me2-server/e2e + E2E_CLASSIC_PROVIDER=1 python3 -m unittest tests.test_classic_provider_e2e -v + +OPENROUTER_API_KEY is taken from the environment or ``code4me2-server/.env`` and is +never printed. Set E2E_KEEP_STACK=1 to leave the stack running for inspection. +""" +import json +import os +import unittest +import uuid +from pathlib import Path + +from code4me_e2e import stack +from code4me_e2e.config import load_scenario +from code4me_e2e.http import HttpClient, HttpResponse + +KEY_NAME = "OPENROUTER_API_KEY" +SERVER_ENV_FILE = Path(__file__).resolve().parents[2] / ".env" +DEFAULT_PROVIDER_ROWS = {1: "completion", 3: "chat"} +# Seeded ids (plugin version, "manual" trigger, a language); the plugin sends the same shape. +TELEMETRY = {"version_id": 1, "trigger_type_id": 1, "language_id": 1} +STORE_FLAGS = {"store_context": True, "store_contextual_telemetry": True, + "store_behavioral_telemetry": True} + + +def openrouter_key() -> str: + """The key from the environment, else from code4me2-server/.env; never printed.""" + key = os.environ.get(KEY_NAME, "").strip() + if not key and SERVER_ENV_FILE.is_file(): + for line in SERVER_ENV_FILE.read_text(encoding="utf-8").splitlines(): + name, _, value = line.partition("=") + if name.strip() == KEY_NAME: + key = value.strip().strip("\"'") + return key + + +def expect(response: HttpResponse, status: int, what: str): + if response.status != status: + raise AssertionError(f"{what}: HTTP {response.status}: {response.text[:300]}") + return response.json + + +@unittest.skipUnless( + os.environ.get("E2E_CLASSIC_PROVIDER") == "1", + "live test (Docker stack + real OpenRouter calls): set E2E_CLASSIC_PROVIDER=1", +) +class ClassicProviderEndToEnd(unittest.TestCase): + @classmethod + def setUpClass(cls): + key = openrouter_key() + if not key: + raise unittest.SkipTest(f"{KEY_NAME} is not set in the environment or {SERVER_ENV_FILE}") + # Compose interpolation hands the key to the disposable backend only. + os.environ[KEY_NAME] = key + cls.scenario = load_scenario( + overrides=[ + "stack.project_name=code4me-e2e-classic", + "stack.backend_port=28108", + "stack.db_port=25532", + "stack.redis_port=26479", + "stack.stub_port=29099", + ] + ) + stack.up(cls.scenario) + stack.wait_until_ready(cls.scenario) + cls.client = cls.plugin_like_client() + + @classmethod + def tearDownClass(cls): + if os.environ.get("E2E_KEEP_STACK") != "1": + stack.down(cls.scenario) + + @classmethod + def plugin_like_client(cls) -> HttpClient: + """A signed-in user with a session and an active project, as the plugin has.""" + account = cls.scenario.participant + client = HttpClient(cls.scenario.base_url, label="classic", timeout=90.0) + config_id = int(stack.psql(cls.scenario, "SELECT config_id FROM config ORDER BY config_id LIMIT 1;")) + created = client.post( + "/api/user/create", + {"email": account.email, "name": account.name, "password": account.password, + "config_id": config_id}, + ) + if created.status not in (201, 409): + raise AssertionError(f"creating the user: HTTP {created.status}: {created.text[:300]}") + expect(client.post("/api/user/authenticate", + {"email": account.email, "password": account.password}), 200, "login") + expect(client.get("/api/session/acquire"), 200, "session acquire") + project = expect(client.post("/api/project/create", {"project_name": "classic-e2e"}), + 201, "project create") + expect(client.put("/api/project/activate", {"project_id": project["project_token"]}), + 200, "project activate") + return client + + def assert_served_by_provider(self, kind: str) -> None: + logs = stack.backend_logs(self.scenario, tail=500) + self.assertIn(f":provider:{kind}:", logs, "the provider-backed row was not used") + self.assertIn("openrouter.ai/api/v1/chat/completions", logs) + self.assertNotIn("Loading model with cache directory", logs, "a local model was loaded") + + def test_default_rows_are_provider_backed_and_the_rest_stay_local(self): + rows = stack.psql(self.scenario, "SELECT model_id, model_parameters FROM model_name ORDER BY model_id;") + params = {int(model_id): json.loads(raw) for model_id, raw in + (line.split("|", 1) for line in rows.splitlines())} + for model_id, kind in DEFAULT_PROVIDER_ROWS.items(): + self.assertEqual(params[model_id].get("provider"), "openai_compatible", params[model_id]) + self.assertEqual(params[model_id].get("kind"), kind) + self.assertEqual(params[model_id].get("api_key_ref"), KEY_NAME) + local = [model_id for model_id, row in params.items() if "provider" not in row] + self.assertEqual(sorted(local), sorted(set(params) - set(DEFAULT_PROVIDER_ROWS))) + + def test_completion_is_answered_by_the_provider(self): + body = expect(self.client.post("/api/completion/request", { + "model_ids": [1], + "context": {"prefix": 'def fibonacci(n):\n """Return the n-th Fibonacci number."""\n', + "suffix": "\n\nprint(fibonacci(10))\n", "file_name": "fib.py"}, + "contextual_telemetry": TELEMETRY, + "behavioral_telemetry": {}, + **STORE_FLAGS, + }), 200, "completion request") + + [item] = body["data"]["completions"] + self.assertEqual(item.get("model_id"), 1, item) + self.assertTrue(item["completion"].strip(), item) + self.assert_served_by_provider("completion") + + def test_chat_is_answered_by_the_provider(self): + body = expect(self.client.post("/api/chat/request", { + "model_ids": [3], + "chat_id": str(uuid.uuid4()), + "messages": [["user", "In one short sentence, what is a Python list comprehension?"]], + "context": {"prefix": "", "suffix": "", "file_name": "scratch.py"}, + "contextual_telemetry": TELEMETRY, + "behavioral_telemetry": {}, + "web_enabled": False, + **STORE_FLAGS, + }), 200, "chat request") + + [answer] = body["history"][0]["assistant_responses"] + self.assertEqual(answer.get("model_id"), 3, answer) + self.assertTrue(answer["completion"].strip(), answer) + self.assert_served_by_provider("chat") + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e/tests/test_completion_regressions.py b/e2e/tests/test_completion_regressions.py new file mode 100644 index 0000000..2ac8684 --- /dev/null +++ b/e2e/tests/test_completion_regressions.py @@ -0,0 +1,87 @@ +"""Failures observed while running the inherited harness against real agents.""" +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from _support import isolate_prerequisites +from code4me_e2e import acp, acp_probe, native_agents, real_agents, suite, workflow +from code4me_e2e.config import load_scenario +from code4me_e2e.steps import StepResult +from test_real_agents import _fake_agent_script, _make_executable, _write_registry + + +class CompletionRegressionTest(unittest.TestCase): + def test_error_text_with_end_turn_cannot_pass_as_the_expected_model_answer(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + executable = _make_executable(root, "goose", _fake_agent_script("ok", "goose")) + result = acp_probe.run_probe("goose", run_dir=root / "run", executable=str(executable), + expected_substring="unique-provider-answer", timeout=5) + self.assertFalse(result.passed) + self.assertIn("expected provider answer", result.detail) + + def test_expected_answer_does_not_replace_the_live_provider_receipt(self): + with tempfile.TemporaryDirectory() as tmp, patch.object( + acp_probe, "run_probe", return_value=real_agents.ProbeResult.ok("goose") + ): + result = native_agents.probe("goose", Path(tmp)) + self.assertFalse(result.passed) + self.assertIn("No request", result.detail) + + def test_agent_credentials_in_an_existing_home_cannot_be_consumed(self): + with tempfile.TemporaryDirectory() as tmp: + home = Path(tmp) / "existing" + home.mkdir() + (home / "config").write_text("synthetic existing configuration") + with self.assertRaisesRegex(ValueError, "must be empty"): + real_agents.isolated_home(Path(tmp), "codex", str(home)) + + def test_digest_in_child_arguments_is_not_a_proxy_pin(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + executable = _make_executable(root, "proxy", "#!/bin/sh\necho test\n") + _write_registry(root, {"Code4Me Research Proxy": { + "command": str(executable), + "args": ["--agent-cmd", str(executable), "--agent-digest", "a" * 64], + }}) + with self.assertRaisesRegex(RuntimeError, "no --agent-digest before"): + acp.entry_identity(root) + + def test_missing_layer_evidence_cannot_pass_even_with_zero_exit(self): + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual("FAIL", suite.layer_outcome(Path(tmp), step_id="ui_test", rc=0)[0]) + + def setUp(self): + isolate_prerequisites(self) + + def test_backend_only_starts_at_account_creation_not_bootstrap(self): + def execute(scenario, **kwargs): + workflow._record_plugin_step(Path(kwargs["run_dir"]), scenario, StepResult("verify", "PASS", 1)) + return 0 + with tempfile.TemporaryDirectory() as tmp, patch.object(workflow, "run_workflow", side_effect=execute) as run, \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(0, suite.run(load_scenario(), layer="backend", run_dir=tmp, keep_stack=True)) + self.assertNotIn("from_step", run.call_args.kwargs) + + def test_unexpected_ide_exception_still_runs_independent_agents_and_browser(self): + attempted = [] + def execute(name, scenario, run_path, executables=None): + attempted.append(name) + if name == "ui": + raise RuntimeError("fixture launch crashed") + workflow._record_plugin_step(run_path, scenario, + StepResult(suite.LAYER_STEP_IDS[name], "PASS", 1)) + return 0 + with tempfile.TemporaryDirectory() as tmp, patch.object(suite, "_run_layer", side_effect=execute), \ + patch.object(suite.report, "capture_backend_logs"), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(1, suite.run(load_scenario(), run_dir=tmp, keep_stack=True)) + self.assertEqual(["ui", "agents", "browser"], attempted) + built = json.loads((Path(tmp) / "report.json").read_text()) + self.assertEqual("suite", built["failed_step"]) + self.assertEqual(["FAIL", "NOT_ATTEMPTED", "NOT_ATTEMPTED", "PASS", "PASS"], + [item["status"] for item in built["layers"]]) diff --git a/e2e/tests/test_harness.py b/e2e/tests/test_harness.py new file mode 100644 index 0000000..3a38d9f --- /dev/null +++ b/e2e/tests/test_harness.py @@ -0,0 +1,294 @@ +"""Regression tests for false passes, isolation and resumable orchestration.""" +import contextlib +import io +import json +import os +import socket +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +from _support import isolate_prerequisites +from code4me_e2e import cli, process, report, steps, suite, ui, workflow +from code4me_e2e.config import ScenarioError, load_scenario +from code4me_e2e.steps import Ctx, StepResult +from code4me_e2e.stub_provider import StubProvider + + +class HarnessTest(unittest.TestCase): + def setUp(self): + # Orchestration tests must never start a real browser or installed agent, + # nor run (or provision) real host prerequisites. + isolate_prerequisites(self) + for module, step in ((suite.browser, "browser_test"), (suite.native_agents, "agents_test")): + def record(scenario, run_path, step=step, **_kwargs): + workflow._record_plugin_step(Path(run_path), scenario, StepResult(step, "PASS", 1)) + return 0 + patcher = patch.object(module, "run", side_effect=record) + patcher.start() + self.addCleanup(patcher.stop) + + def test_gradle_success_requires_executed_unskipped_tests(self): + for evidence in ({}, {"tests": 0}, {"tests": 1, "skipped": 1}, + {"tests": 1, "failures": 1}, {"tests": 1, "errors": 1}): + with self.subTest(evidence=evidence), self.assertRaises(RuntimeError): + ui.require_test_evidence(0, evidence) + with self.assertRaises(RuntimeError): + ui.require_test_evidence(1, {"tests": 1}) + + def test_ui_gate_requires_every_step_once(self): + steps = [{"id": name, "status": "PASS"} for name in ui.EXPECTED_UI_STEPS] + ui.require_test_evidence(0, {"tests": 1}, {"steps": steps}) + for invalid in (steps[:-1], steps + steps[:1], + [{**step, "status": "BLOCKED"} for step in steps]): + with self.subTest(steps=invalid), self.assertRaises(RuntimeError): + ui.require_test_evidence(0, {"tests": 1}, {"steps": invalid}) + + def test_resume_authenticates_once_instead_of_reusing_invalidated_login(self): + scenario = load_scenario() + state = {"accounts": {"participant": {}}, "cookies": {"participant": {"auth_token": "expired"}}} + with patch("code4me_e2e.steps.HttpClient") as http: + http.return_value.post.return_value.status = 200 + ctx = Ctx(scenario, state, Path("unused"), fresh_login=True) + self.assertIs(ctx.client("participant"), ctx.client("participant")) + http.return_value.post.assert_called_once_with("/api/user/authenticate", { + "email": scenario.participant.email, "password": scenario.participant.password, + }) + + def test_report_replaces_only_its_own_step_and_redacts_secrets(self): + with tempfile.TemporaryDirectory() as directory: + path, scenario = Path(directory), load_scenario() + workflow._record_plugin_step(path, scenario, StepResult("plugin_test", "PASS", 1)) + workflow._record_plugin_step(path, scenario, StepResult("cleanup", "FAIL", 1, {"session_capability": "secret"})) + result = json.loads((path / "report.json").read_text()) + self.assertEqual([s["id"] for s in result["steps"]], ["plugin_test", "cleanup"]) + self.assertNotIn('"secret"', (path / "report.json").read_text()) + + def test_exception_after_passing_layer_cannot_return_success(self): + with tempfile.TemporaryDirectory() as directory, \ + patch.object(workflow, "run_plugin_test", return_value=0), \ + patch.object(workflow, "run_workflow", side_effect=RuntimeError("backend startup failed")), \ + patch.object(report, "capture_backend_logs"), \ + patch.object(suite.stack, "down") as down, \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + workflow._record_plugin_step(Path(directory), load_scenario(), StepResult("plugin_test", "PASS", 1)) + rc = suite.run(load_scenario(), layer="plugin", run_dir=directory) + self.assertEqual(1, rc) + down.assert_not_called() + self.assertEqual("suite", json.loads((Path(directory) / "report.json").read_text())["failed_step"]) + + def test_ui_failure_stops_dependent_layers(self): + with tempfile.TemporaryDirectory() as directory, \ + patch.object(ui, "run_ui_test", return_value=1), \ + patch.object(workflow, "run_plugin_test") as plugin, \ + patch.object(workflow, "run_workflow") as backend, \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(1, suite.run(load_scenario(), run_dir=directory)) + plugin.assert_not_called() + backend.assert_not_called() + + def test_cleanup_failure_is_a_failed_gate(self): + with tempfile.TemporaryDirectory() as directory, \ + patch.object(workflow, "run_workflow", return_value=0), \ + patch.object(suite.stack, "down", side_effect=RuntimeError("docker unavailable")), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + workflow._record_plugin_step(Path(directory), load_scenario(), StepResult("verify", "PASS", 1)) + self.assertEqual(1, suite.run(load_scenario(), layer="backend", run_dir=directory)) + self.assertEqual("cleanup", json.loads((Path(directory) / "report.json").read_text())["failed_step"]) + + def test_disposable_project_name_cannot_select_developer_stack(self): + with self.assertRaises(ScenarioError): + load_scenario(overrides=["stack.project_name=code4me2-server"]) + args = cli.build_parser().parse_args(["stack", "down", "--project-name", "code4me2-server"]) + with self.assertRaises(ScenarioError): + cli._load(args) + self.assertEqual("code4me-e2e-ci", load_scenario(overrides=["stack.project_name=code4me-e2e-ci"]).stack.project_name) + + def test_invalid_ports_and_non_object_scenario_fail_before_startup(self): + for override in ("stack.stub_port=0", "stack.db_port=28008", "stack.redis_port=false"): + with self.subTest(override=override), self.assertRaises(ScenarioError): + load_scenario(overrides=[override]) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "scenario.json" + path.write_text("[]") + with self.assertRaises(ScenarioError): + load_scenario(str(path)) + + def test_stub_does_not_change_the_registered_port_when_it_is_busy(self): + with socket.socket() as occupied: + occupied.bind(("127.0.0.1", 0)) + occupied.listen() + with self.assertRaises(OSError): + StubProvider().start(occupied.getsockname()[1]) + + def test_stub_listens_on_the_scenario_host(self): + self.assertEqual("127.0.0.1", load_scenario().stack.stub_host) + for override in ("stack.stub_host=localhost", "stack.stub_host=1"): + with self.subTest(override=override), self.assertRaises(ScenarioError): + load_scenario(overrides=[override]) + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + ctx = Ctx(load_scenario(overrides=[f"stack.stub_port={port}", "stack.stub_host=0.0.0.0"]), {}, Path(".")) + try: + self.assertEqual(port, ctx.ensure_stub()) + self.assertEqual("0.0.0.0", ctx.stub._server.server_address[0]) + # The harness itself still reaches it on loopback. + socket.create_connection(("127.0.0.1", port), timeout=5).close() + finally: + ctx.stub.stop() + + def test_timeout_reaps_the_owned_process(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + code = "import os,pathlib,time; pathlib.Path('pid').write_text(str(os.getpid())); time.sleep(60)" + with self.assertRaises(subprocess.TimeoutExpired): + process.run([sys.executable, "-c", code], cwd=path, log_path=path / "process.log", timeout=1) + pid = int((path / "pid").read_text()) + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_a_foreign_blocked_step_does_not_fail_a_passing_workflow(self): + """Regression: the gate's prerequisites record (e.g. no Goose) broke the IDE prefix.""" + passing = {step_id: (lambda ctx: {}) for step_id in steps.STEP_ORDER} + with tempfile.TemporaryDirectory() as directory, \ + patch.dict(workflow.STEPS, passing), \ + patch.object(workflow, "_own_stack", return_value=False), \ + patch.object(report, "capture_backend_logs") as capture, \ + contextlib.redirect_stdout(io.StringIO()): + workflow._record_plugin_step(Path(directory), load_scenario(), StepResult( + "prerequisites", "BLOCKED", 1, {}, "goose: install Goose")) + rc = workflow.run_workflow(load_scenario(), only="doctor,create_accounts", run_dir=directory) + built = json.loads((Path(directory) / "report.json").read_text()) + self.assertEqual(0, rc) + capture.assert_not_called() + self.assertEqual("prerequisites", built["blocked_by"], "the report still shows the blocked check") + failing = dict(passing, create_accounts=lambda ctx: (_ for _ in ()).throw(steps.StepFailure("boom"))) + with patch.dict(workflow.STEPS, failing): + self.assertEqual(1, workflow.run_workflow(load_scenario(), only="create_accounts", run_dir=directory)) + + def test_only_an_unhandled_500_is_reported_as_an_application_bug(self): + """Regression: a deliberate 503 (an unavailable store) was reported as an application bug.""" + bodies = {503: {"detail": "telemetry store is unavailable"}, 502: "502 Bad Gateway"} + + def failing_after(*statuses): + def step(ctx): + for status in statuses: + ctx.exchange_log.append({ + "client": "participant", "method": "POST", "path": f"/api/{status}", + "status": status, "response": bodies.get(status), + }) + raise steps.StepFailure("boom") + return step + + ctx = Ctx(load_scenario(), {}, Path(".")) + with patch.dict(workflow.STEPS, {"handled": failing_after(502, 503), "crashed": failing_after(500, 503)}): + workflow._run_one(ctx, "handled") + workflow._run_one(ctx, "crashed") + handled, crashed = ctx.findings + self.assertEqual("environment", handled["severity"]) + self.assertIn("HTTP 503 from POST /api/503 (telemetry store is unavailable)", handled["message"]) + self.assertNotIn("application bug", handled["message"]) + self.assertEqual("bug", crashed["severity"]) + self.assertIn("HTTP 500 from POST /api/500", crashed["message"]) + + # Resumed after the environment was fixed, the same step now crashes. + with patch.dict(workflow.STEPS, {"handled": failing_after(500)}): + workflow._run_one(ctx, "handled") + self.assertEqual("bug", ctx.findings[-1]["severity"]) + self.assertIn("step handled received HTTP 500", ctx.findings[-1]["message"]) + + def test_multipart_body_carries_the_manifest_and_archive(self): + from code4me_e2e.http import encode_multipart + body, content_type = encode_multipart( + {"manifest": '{"artifacts": []}'}, [("archives", "agent.zip", b"PK\x03\x04agent")] + ) + boundary = content_type.split("boundary=", 1)[1] + self.assertTrue(content_type.startswith("multipart/form-data; boundary=")) + self.assertIn(b'name="manifest"', body) + self.assertIn(b'{"artifacts": []}', body) + self.assertIn(b'filename="agent.zip"', body) + self.assertIn(b"PK\x03\x04agent", body) + self.assertTrue(body.rstrip().endswith(f"--{boundary}--".encode())) + + def test_register_release_imports_the_producer_manifest_and_archive(self): + with tempfile.TemporaryDirectory() as directory: + manifest = Path(directory) / "native-macos-arm64.json" + archive = Path(directory) / "code4me-agent-macos-arm64.zip" + manifest.write_text(json.dumps({"artifacts": [{"archive": archive.name, "sha256": "a" * 64}]})) + archive.write_bytes(b"PK\x03\x04agent") + ctx = Ctx(load_scenario(), {"accounts": {}, "admin_user_id": "admin"}, Path(directory)) + with patch("code4me_e2e.steps.runtime.agent_release", return_value=(manifest, archive)), \ + patch.object(Ctx, "client") as client: + client.return_value.post_multipart.return_value = Mock( + status=201, + json={ + "created": True, + "release": {"release_id": "code4me-agent-1.0.0-abc"}, + "verified_artifacts": [{"verified": True}], + }, + ) + result = steps.step_register_release(ctx) + self.assertEqual("code4me-agent-1.0.0-abc", result["release_id"]) + self.assertEqual("code4me-agent-1.0.0-abc", ctx.state["release_id"]) + self.assertEqual("code4me-agent-1.0.0-abc", ctx.scenario.agent.release_id) + self.assertEqual("a" * 64, ctx.scenario.agent.artifact_digest) + call = client.return_value.post_multipart.call_args + self.assertEqual("/api/research/agents/releases/import", call.args[0]) + self.assertEqual(["archives"], [name for name, _, _ in call.kwargs["files"]]) + self.assertEqual(archive.name, call.kwargs["files"][0][1]) + self.assertIn('"archive"', call.kwargs["fields"]["manifest"]) + + def test_qualify_release_requires_producer_test_results(self): + ctx = Ctx(load_scenario(), {"release_id": "r1", "accounts": {}}, Path(".")) + with patch.object(Ctx, "client") as client: + client.return_value.get.return_value = Mock( + status=200, json={"release": {"status": "UNQUALIFIED"}} + ) + with self.assertRaises(steps.StepFailure): + steps.step_qualify_release(ctx) + client.return_value.get.return_value = Mock( + status=200, json={"release": {"status": "QUALIFIED"}} + ) + self.assertEqual("QUALIFIED", steps.step_qualify_release(ctx)["derived_status"]) + + + def test_send_message_names_its_research_session(self): + # The plugin layer's Kotlin fixture opens a second active session for the + # same account; without an explicit id the server refuses to guess. + state = {"accounts": {}, "acp_token": "token", "research_session_id": "rs-1"} + ctx = Ctx(load_scenario(), state, Path(".")) + with patch.object(Ctx, "client") as client: + client.return_value.post.return_value = Mock( + status=409, json={"detail": {"code": "RESEARCH_CONTEXT_AMBIGUOUS"}}, text="" + ) + with self.assertRaises(steps.StepFailure): + steps.step_send_message(ctx) + path, body = client.return_value.post.call_args.args[:2] + self.assertEqual("/api/acp/runs", path) + self.assertEqual("rs-1", body["research_session_id"]) + + +class AgentCacheSeedTest(unittest.TestCase): + def test_seed_places_the_archive_where_the_plugin_cache_looks(self): + import hashlib + from code4me_e2e import runtime + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive = root / "code4me-agent-macos-arm64.zip" + archive.write_bytes(b"PK\x03\x04agent") + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + target = root / "system/code4me/runtimes/code4me-agent/archives" / f"{digest}.zip" + + self.assertEqual(digest, runtime.seed_agent_cache(root / "system", archive)) + self.assertEqual(archive.read_bytes(), target.read_bytes()) + target.write_bytes(b"corrupted") + self.assertEqual(digest, runtime.seed_agent_cache(root / "system", archive)) + self.assertEqual(archive.read_bytes(), target.read_bytes()) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e/tests/test_native_gateway_probe.py b/e2e/tests/test_native_gateway_probe.py new file mode 100644 index 0000000..a5434b9 --- /dev/null +++ b/e2e/tests/test_native_gateway_probe.py @@ -0,0 +1,118 @@ +"""The Goose probe runs in the research gateway shape and understands a 402.""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +from code4me_e2e import acp_probe, native_agents, real_agents +from code4me_e2e.agent_provider import GATEWAY_CHAT_ROUTE +from code4me_e2e.config import AgentProbeReason + + +def _post(env, *, bearer, stream=False): + """Post one Chat Completions request the way Goose would, from its env.""" + url = env["OPENAI_HOST"].rstrip("/") + "/" + env["OPENAI_BASE_PATH"] + request = Request( + url, data=json.dumps({"model": env.get("GOOSE_MODEL"), "stream": stream, "messages": []}).encode("utf-8"), + headers={"Content-Type": "application/json", **({"Authorization": f"Bearer {bearer}"} if bearer else {})}, + method="POST", + ) + try: + with urlopen(request, timeout=5) as response: # noqa: S310 - loopback fixture + return response.status + except HTTPError as error: + return error.code + + +class GatewayShapeProbeTest(unittest.TestCase): + def _run(self, side_effect, **kwargs): + with tempfile.TemporaryDirectory() as tmp, patch.object(acp_probe, "run_probe", side_effect=side_effect): + return native_agents.probe("goose", Path(tmp), **kwargs) + + def test_goose_env_is_the_gateway_shape_and_the_home_is_poisoned(self): + seen = {} + + def fake_run_probe(framework, **kwargs): + env = kwargs["env_extra"] + seen.update(env) + with tempfile.TemporaryDirectory() as home: + kwargs["prepare_home"](Path(home)) + seen["poisoned"] = (Path(home) / ".config" / "goose" / "config.yaml").read_text() + self.assertEqual(200, _post(env, bearer=env["OPENAI_API_KEY"])) + return real_agents.ProbeResult.ok(framework) + + result = self._run(fake_run_probe) + self.assertTrue(result.passed, result.detail) + self.assertEqual("openai", seen["GOOSE_PROVIDER"]) + self.assertEqual(GATEWAY_CHAT_ROUTE.lstrip("/"), seen["OPENAI_BASE_PATH"]) + self.assertTrue(seen["OPENAI_API_KEY"].startswith("c4m-")) + self.assertNotIn("OPENAI_BASE_URL", seen) + self.assertTrue(seen["GOOSE_PATH_ROOT"].endswith("goose-root")) + self.assertIn("GOOSE_PROVIDER: anthropic", seen["poisoned"]) + self.assertTrue(result.protocol["gateway_shape"]) + self.assertTrue(all(item["auth_matched"] for item in result.protocol["provider_requests"])) + + def test_a_missing_bearer_blocks_the_probe(self): + def fake_run_probe(framework, **kwargs): + self.assertEqual(200, _post(kwargs["env_extra"], bearer=None)) + return real_agents.ProbeResult.ok(framework) + + result = self._run(fake_run_probe) + self.assertEqual("BLOCKED", result.status) + self.assertIn("gateway bearer", result.detail) + + def test_quota_mode_passes_only_on_a_typed_quota_block_after_one_request(self): + def one_refused_request(framework, **kwargs): + env = kwargs["env_extra"] + self.assertEqual(402, _post(env, bearer=env["OPENAI_API_KEY"], stream=True)) + return real_agents.ProbeResult.blocked( + framework, AgentProbeReason.QUOTA, "Credits exhausted: quota_exhausted" + ) + + result = self._run(one_refused_request, quota_exhausted=True) + self.assertTrue(result.passed, result.detail) + self.assertTrue(result.protocol["quota_exhausted_mode"]) + + def normal_turn(framework, **kwargs): + # Goose 1.51: the reply call plus the session-description call. + env = kwargs["env_extra"] + for _ in range(2): + _post(env, bearer=env["OPENAI_API_KEY"]) + return real_agents.ProbeResult.blocked( + framework, AgentProbeReason.QUOTA, + "Please check your account with your provider to add more credits (credits_exhausted)", + ) + + self.assertTrue(self._run(normal_turn, quota_exhausted=True).passed) + + def retry_storm(framework, **kwargs): + env = kwargs["env_extra"] + for _ in range(3): + _post(env, bearer=env["OPENAI_API_KEY"]) + return real_agents.ProbeResult.blocked(framework, AgentProbeReason.QUOTA, "quota") + + result = self._run(retry_storm, quota_exhausted=True) + self.assertEqual("BLOCKED", result.status) + self.assertIn("no retries", result.detail) + + def answered_anyway(framework, **kwargs): + _post(kwargs["env_extra"], bearer=kwargs["env_extra"]["OPENAI_API_KEY"]) + return real_agents.ProbeResult.ok(framework) + + result = self._run(answered_anyway, quota_exhausted=True) + self.assertEqual("BLOCKED", result.status) + + def test_classifier_recognises_the_gateway_refusal(self): + for text in ( + "Credits exhausted: Your study's AI budget is used up (quota_exhausted)", + # Goose 1.51's own ACP wording for the gateway's 402. + "Please check your account with your provider to add more credits, then resend your message", + '{"reason": "credits_exhausted"}', + ): + self.assertEqual(AgentProbeReason.QUOTA, real_agents.classify_failure(text), text) diff --git a/e2e/tests/test_prereqs.py b/e2e/tests/test_prereqs.py new file mode 100644 index 0000000..c84a3d4 --- /dev/null +++ b/e2e/tests/test_prereqs.py @@ -0,0 +1,592 @@ +"""Host prerequisite checks and provisioning decisions, without touching the host. + +Every external effect (docker, colima, npm, pip, ps, ports) goes through +patched helpers; these tests never start a daemon, build an image, install a +package or signal a process. +""" +from __future__ import annotations + +import contextlib +import io +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from code4me_e2e import cli, prereqs, real_agents +from code4me_e2e.config import load_scenario + + +class _Calls: + """Records every command a check would run and answers from a script.""" + + def __init__(self, answers): + self.answers = answers + self.commands = [] + + def run(self, command, **_kwargs): + self.commands.append(list(command)) + for prefix, answer in self.answers: + if list(command[: len(prefix)]) == list(prefix): + return answer(command) if callable(answer) else answer + raise AssertionError(f"unexpected command {command}") + + def ran(self, *prefix) -> bool: + return any(command[: len(prefix)] == list(prefix) for command in self.commands) + + +class DockerCheckTest(unittest.TestCase): + def setUp(self): + patcher = patch.object(prereqs.stack, "compose_command", return_value=["docker", "compose"]) + patcher.start() + self.addCleanup(patcher.stop) + + def test_reachable_daemon_is_ready_without_starting_anything(self): + calls = _Calls([(["docker", "info"], (0, "29.2.1"))]) + with patch.object(prereqs, "_run", side_effect=calls.run), \ + patch.object(prereqs.shutil, "which", side_effect=lambda name: f"/bin/{name}"): + check = prereqs.check_docker(True) + self.assertEqual(prereqs.READY, check.status) + self.assertFalse(calls.ran("/bin/colima")) + + def test_down_daemon_starts_colima_only_when_provisioning(self): + state = {"started": False} + + def info(_command): + return (0, "29.2.1") if state["started"] else (1, "Cannot connect to the Docker daemon") + + def colima(command, log_name, **_kwargs): + state["started"] = True + return 0, Path("/tmp/colima-start.log") + + calls = _Calls([(["docker", "info"], info)]) + with patch.object(prereqs, "_run", side_effect=calls.run), \ + patch.object(prereqs, "_run_logged", side_effect=colima) as logged, \ + patch.object(prereqs, "_colima_profile", return_value="default"), \ + patch.object(prereqs.shutil, "which", side_effect=lambda name: f"/bin/{name}"), \ + contextlib.redirect_stderr(io.StringIO()): + checked = prereqs.check_docker(False) + self.assertEqual(prereqs.MISSING, checked.status) + logged.assert_not_called() + provisioned = prereqs.check_docker(True) + self.assertEqual(prereqs.PROVISIONED, provisioned.status) + self.assertEqual(["/bin/colima", "start"], logged.call_args.args[0]) + + def test_down_daemon_without_colima_is_blocked_with_a_remediation(self): + calls = _Calls([(["docker", "info"], (1, "Cannot connect"))]) + with patch.object(prereqs, "_run", side_effect=calls.run), \ + patch.object(prereqs.shutil, "which", side_effect=lambda name: None if name == "colima" else f"/bin/{name}"): + check = prereqs.check_docker(True) + self.assertEqual(prereqs.BLOCKED, check.status) + self.assertTrue(check.remediation) + + def test_colima_is_never_started_behind_a_non_colima_context(self): + calls = _Calls([(["docker", "info"], (1, "Cannot connect")), + (["docker", "context", "show"], (0, "desktop-linux\n"))]) + with patch.dict(os.environ, {}, clear=False), \ + patch.object(prereqs, "_run", side_effect=calls.run), \ + patch.object(prereqs, "_run_logged") as logged, \ + patch.object(prereqs.shutil, "which", side_effect=lambda name: f"/bin/{name}"): + os.environ.pop("DOCKER_HOST", None) + check = prereqs.check_docker(True) + self.assertEqual(prereqs.BLOCKED, check.status) + logged.assert_not_called() + + def test_the_colima_profile_follows_the_docker_endpoint(self): + with patch.dict(os.environ, {"DOCKER_HOST": "unix:///Users/x/.colima/dev/docker.sock"}): + self.assertEqual("dev", prereqs._colima_profile()) + with patch.dict(os.environ, {"DOCKER_HOST": "tcp://127.0.0.1:2375"}): + self.assertIsNone(prereqs._colima_profile()) + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("DOCKER_HOST", None) + for name, expected in (("colima", "default"), ("colima-dev", "dev"), ("default", None)): + with patch.object(prereqs, "_run", return_value=(0, name + "\n")): + self.assertEqual(expected, prereqs._colima_profile(), name) + started = [] + with patch.object(prereqs, "_docker_reachable", side_effect=[(False, "down"), (True, "29")]), \ + patch.object(prereqs, "_colima_profile", return_value="dev"), \ + patch.object(prereqs, "_run_logged", side_effect=lambda command, *a, **k: started.append(command) or (0, Path("/tmp/l"))), \ + patch.object(prereqs.shutil, "which", side_effect=lambda name: f"/bin/{name}"), \ + contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(prereqs.PROVISIONED, prereqs.check_docker(True).status) + self.assertEqual([["/bin/colima", "start", "dev"]], started) + + def test_missing_cli_is_blocked(self): + with patch.object(prereqs.shutil, "which", return_value=None): + check = prereqs.check_docker(True) + self.assertEqual(prereqs.BLOCKED, check.status) + self.assertIn("docker CLI", check.detail) + + +class BackendImageTest(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.server = Path(self._tmp.name) + (self.server / "Dockerfile.cpu").write_text("FROM scratch\n") + (self.server / "requirements.txt").write_text("fastapi==1\n") + patcher = patch.object(prereqs, "_server_root", return_value=self.server) + patcher.start() + self.addCleanup(patcher.stop) + self.scenario = load_scenario() + + def _check(self, answers, provision=True): + calls = _Calls(answers) + with patch.object(prereqs, "_run", side_effect=calls.run), \ + patch.object(prereqs, "_run_logged", return_value=(0, Path("/tmp/build.log"))) as logged, \ + contextlib.redirect_stderr(io.StringIO()): + check = prereqs.check_backend_image(provision, scenario=self.scenario) + return check, calls, logged + + def test_harness_label_matching_the_inputs_is_ready(self): + label = json.dumps({prereqs.IMAGE_LABEL: prereqs.image_fingerprint(self.server)}) + check, calls, logged = self._check([(["docker", "image", "inspect"], (0, label))]) + self.assertEqual(prereqs.READY, check.status) + logged.assert_not_called() + self.assertFalse(calls.ran("docker", "run")) + + def test_unlabelled_image_is_judged_by_its_requirements_content(self): + check, _, logged = self._check([ + (["docker", "image", "inspect"], (0, "{}")), + (["docker", "run"], (0, "fastapi==1\n")), + ]) + self.assertEqual(prereqs.READY, check.status) + logged.assert_not_called() + + def test_changed_requirements_rebuild_with_the_fingerprint_label(self): + check, _, logged = self._check([ + (["docker", "image", "inspect"], (0, "{}")), + (["docker", "run"], (0, "fastapi==0\n")), + ]) + self.assertEqual(prereqs.PROVISIONED, check.status) + command = logged.call_args.args[0] + self.assertEqual(["docker", "build", "-f", "Dockerfile.cpu", "-t", self.scenario.stack.image], command[:6]) + self.assertIn(f"{prereqs.IMAGE_LABEL}={prereqs.image_fingerprint(self.server)}", command) + + def test_stale_label_and_missing_image_rebuild_but_check_mode_only_reports(self): + stale = json.dumps({prereqs.IMAGE_LABEL: "0" * 64}) + check, _, logged = self._check([(["docker", "image", "inspect"], (0, stale))], provision=False) + self.assertEqual(prereqs.MISSING, check.status) + logged.assert_not_called() + check, _, logged = self._check([(["docker", "image", "inspect"], (1, "No such image"))]) + self.assertEqual(prereqs.PROVISIONED, check.status) + + def test_failed_build_is_blocked(self): + calls = _Calls([(["docker", "image", "inspect"], (1, "No such image"))]) + with patch.object(prereqs, "_run", side_effect=calls.run), \ + patch.object(prereqs, "_run_logged", return_value=(1, Path("/tmp/build.log"))), \ + contextlib.redirect_stderr(io.StringIO()): + check = prereqs.check_backend_image(True, scenario=self.scenario) + self.assertEqual(prereqs.BLOCKED, check.status) + self.assertIn("/tmp/build.log", check.remediation) + + +class NativePythonTest(unittest.TestCase): + def setUp(self): + patcher = patch.object(prereqs, "_native_choice", None) + patcher.start() + self.addCleanup(patcher.stop) + + def test_the_first_candidate_that_imports_the_build_inputs_is_chosen(self): + candidates = [("/srv/.venv/bin/python", "code4me2-server/.venv"), ("/e2e/.venv/bin/python", "e2e/.venv")] + with patch.dict(os.environ, {}, clear=False), \ + patch.object(prereqs, "native_python_candidates", return_value=candidates), \ + patch.object(prereqs, "_native_probe", side_effect=lambda python: (python.startswith("/e2e"), "No module named PyInstaller")), \ + patch.object(prereqs, "_run_logged") as logged: + check = prereqs.check_native_python(False) + self.assertEqual(prereqs.READY, check.status) + self.assertIn("/e2e/.venv/bin/python", check.detail) + self.assertEqual(("/e2e/.venv/bin/python", "e2e/.venv"), prereqs.resolve_native_python()) + logged.assert_not_called() + + def test_an_explicit_interpreter_is_blocked_not_replaced(self): + with patch.object(prereqs, "native_python_candidates", return_value=[("/custom/python", "CODE4ME_E2E_PYTHON")]), \ + patch.object(prereqs, "_native_probe", return_value=(False, "No module named PyInstaller")), \ + patch.object(prereqs, "_run_logged") as logged: + check = prereqs.check_native_python(True) + self.assertEqual(prereqs.BLOCKED, check.status) + logged.assert_not_called() + + def test_provisioning_is_pinned_like_the_release_ci(self): + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(prereqs, "E2E_DIR", Path(tmp) / "e2e"), \ + patch.object(prereqs, "_workspace", return_value=Path(tmp)), \ + patch.object(prereqs, "native_python_candidates", return_value=[]), \ + patch.object(prereqs, "_native_probe", return_value=(True, "")), \ + patch.object(prereqs, "_run_logged", return_value=(0, Path("/tmp/log"))) as logged, \ + contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(prereqs.MISSING, prereqs.check_native_python(False).status) + logged.assert_not_called() + check = prereqs.check_native_python(True) + commands = [call.args[0] for call in logged.call_args_list] + server = Path(tmp) / "code4me2-server" + self.assertEqual(prereqs.PROVISIONED, check.status) + self.assertEqual(["-m", "venv"], commands[0][1:3]) + self.assertIn(str(server / "packaging/requirements-runtime.lock"), commands[1]) + self.assertIn(str(server / "packaging/requirements-build.lock"), commands[1]) + self.assertEqual(["-m", "pip", "install", "--no-deps", "-e", str(server)], commands[2][1:]) + + +class RunLoggedTest(unittest.TestCase): + def test_an_interrupted_provisioning_step_leaves_no_process_behind(self): + import _thread + import threading + import uuid + marker = f"code4me-e2e-orphan-{uuid.uuid4().hex}" + with tempfile.TemporaryDirectory() as tmp, patch.object(prereqs, "SETUP_LOG_DIR", Path(tmp)): + timer = threading.Timer(1.0, _thread.interrupt_main) + timer.start() + try: + with self.assertRaises(KeyboardInterrupt): + prereqs._run_logged(["/bin/sh", "-c", f"sleep 30; echo {marker}"], "interrupt.log", timeout=60) + finally: + timer.cancel() + _, listing = prereqs._run(["ps", "-ww", "-eo", "pid=,command="], timeout=30) + self.assertNotIn(marker, listing) + + +class TerminationTest(unittest.TestCase): + def test_sigterm_during_provisioning_leaves_no_process_behind(self): + import signal + import subprocess + import sys + import textwrap + import time + import uuid + marker = f"code4me-e2e-sigterm-{uuid.uuid4().hex}" + with tempfile.TemporaryDirectory() as tmp: + script = textwrap.dedent(f""" + import sys + sys.path.insert(0, {str(prereqs.E2E_DIR)!r}) + from pathlib import Path + from unittest.mock import patch + from code4me_e2e import cli, prereqs + + def fake_main(argv): + prereqs._run_logged(["/bin/sh", "-c", "sleep 30; echo {marker}"], "t.log", timeout=60) + return 0 + + with patch.object(prereqs, "SETUP_LOG_DIR", Path({tmp!r})), \\ + patch.object(cli, "_main", side_effect=fake_main), \\ + patch.object(cli.stack, "E2E_DIR", Path({tmp!r})): + sys.exit(cli.main(["setup"])) + """) + # A script file, not `-c`: only the build process may carry the marker. + script_path = Path(tmp) / "terminate_me.py" + script_path.write_text(script, encoding="utf-8") + child = subprocess.Popen([sys.executable, str(script_path)]) + try: + deadline = time.monotonic() + 20 + while time.monotonic() < deadline and marker not in prereqs._run(["ps", "-ww", "-eo", "command="])[1]: + time.sleep(0.2) + self.assertIn(marker, prereqs._run(["ps", "-ww", "-eo", "command="])[1], "the build never started") + child.send_signal(signal.SIGTERM) + self.assertEqual(143, child.wait(timeout=30)) + finally: + if child.poll() is None: + child.kill() + self.assertNotIn(marker, prereqs._run(["ps", "-ww", "-eo", "command="])[1]) + + +class PortProbeTest(unittest.TestCase): + def test_a_live_listener_is_busy_and_a_closed_port_is_free(self): + import socket + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + port = listener.getsockname()[1] + try: + self.assertFalse(prereqs._port_free(port)) + finally: + listener.close() + self.assertTrue(prereqs._port_free(port)) + + def test_time_wait_leftovers_do_not_count_as_busy(self): + import socket + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + server.listen(1) + port = server.getsockname()[1] + client = socket.create_connection(("127.0.0.1", port), timeout=5) + accepted, _ = server.accept() + # The server side closes first, leaving TIME_WAIT on the listening port. + accepted.close() + client.close() + server.close() + self.assertTrue(prereqs._port_free(port)) + + +class PortsAndIdesTest(unittest.TestCase): + def test_free_ports_are_ready_and_foreign_listeners_block(self): + scenario = load_scenario() + with patch.object(prereqs, "_port_free", return_value=True): + self.assertEqual(prereqs.READY, prereqs.check_stack_ports(True, scenario=scenario).status) + busy = {scenario.stack.stub_port} + with patch.object(prereqs, "_port_free", side_effect=lambda port: port not in busy), \ + patch.object(prereqs, "_port_owner", return_value=" (python 4242)"): + check = prereqs.check_stack_ports(True, scenario=scenario) + self.assertEqual(prereqs.BLOCKED, check.status) + self.assertIn("stack.stub_port", check.detail) + + def test_ports_held_by_the_own_compose_project_are_ready(self): + scenario = load_scenario() + own = {scenario.stack.backend_port, scenario.stack.db_port, scenario.stack.redis_port} + with patch.object(prereqs, "_port_free", side_effect=lambda port: port not in own), \ + patch.object(prereqs.stack, "is_up", return_value=True): + self.assertEqual(prereqs.READY, prereqs.check_stack_ports(True, scenario=scenario).status) + with patch.object(prereqs, "_port_free", side_effect=lambda port: port not in own), \ + patch.object(prereqs.stack, "is_up", return_value=False), \ + patch.object(prereqs, "_port_owner", return_value=""): + self.assertEqual(prereqs.BLOCKED, prereqs.check_stack_ports(True, scenario=scenario).status) + + def test_only_harness_ides_are_selected_and_signalled(self): + runs = prereqs.E2E_DIR / "runs" + listing = "\n".join([ + f"101 java -Duser.home={runs}/20260101T000000Z-aaaaaa/ide-home -Drobot-server.port=5000 idea", + "102 java -Duser.home=/Users/someone -Drobot-server.port=5001 idea", + f"103 java -Duser.home={runs}/20260101T000000Z-bbbbbb/ide-home idea-without-robot", + ]) + with patch.object(prereqs, "_run", return_value=(0, listing)): + self.assertEqual([101], [pid for pid, _ in prereqs._harness_ide_processes()]) + self.assertEqual(prereqs.MISSING, prereqs.check_stale_ides(False).status) + signalled = [] + sequence = [[(101, "idea")], []] + with patch.object(prereqs, "_harness_ide_processes", side_effect=lambda: sequence.pop(0) if sequence else []), \ + patch.object(prereqs.os, "kill", side_effect=lambda pid, sig: signalled.append((pid, sig))): + check = prereqs.check_stale_ides(True) + self.assertEqual(prereqs.PROVISIONED, check.status) + self.assertEqual([(101, prereqs.signal.SIGTERM)], signalled) + + +class NodeAndBrowserTest(unittest.TestCase): + def test_node_version_floor(self): + with patch.object(prereqs.shutil, "which", side_effect=lambda name: f"/bin/{name}"): + with patch.object(prereqs, "_run", return_value=(0, "v22.23.2")): + self.assertEqual(prereqs.READY, prereqs.check_node(True).status) + with patch.object(prereqs, "_run", return_value=(0, "v16.20.0")): + self.assertEqual(prereqs.BLOCKED, prereqs.check_node(True).status) + with patch.object(prereqs.shutil, "which", return_value=None): + self.assertEqual(prereqs.BLOCKED, prereqs.check_node(True).status) + + def test_explicit_browser_python_is_never_replaced(self): + with patch.dict(os.environ, {"CODE4ME_E2E_BROWSER_PYTHON": "/custom/python"}), \ + patch.object(prereqs, "_chromium_ready", return_value=(False, "No module named playwright")), \ + patch.object(prereqs, "_run_logged") as logged: + check = prereqs.check_browser_python(True) + self.assertEqual(prereqs.BLOCKED, check.status) + logged.assert_not_called() + + def test_missing_playwright_provisions_the_pinned_harness_venv(self): + with tempfile.TemporaryDirectory() as tmp, \ + patch.dict(os.environ, {}, clear=False), \ + patch.object(prereqs, "BROWSER_VENV", Path(tmp) / ".browser-venv"), \ + patch.object(prereqs, "_chromium_ready", side_effect=[(False, "missing"), (True, "")]), \ + patch.object(prereqs, "_run_logged", return_value=(0, Path("/tmp/log"))) as logged, \ + contextlib.redirect_stderr(io.StringIO()): + os.environ.pop("CODE4ME_E2E_BROWSER_PYTHON", None) + check = prereqs.check_browser_python(True) + commands = [call.args[0] for call in logged.call_args_list] + self.assertEqual(str(Path(tmp) / ".browser-venv/bin/python"), os.environ["CODE4ME_E2E_BROWSER_PYTHON"]) + self.assertEqual(prereqs.PROVISIONED, check.status) + self.assertIn(f"playwright=={prereqs.PLAYWRIGHT_VERSION}", commands[1]) + self.assertEqual(["-m", "playwright", "install", "chromium"], commands[2][1:]) + + def test_check_mode_never_installs(self): + with patch.dict(os.environ, {}, clear=False), \ + patch.object(prereqs, "_chromium_ready", return_value=(False, "missing")), \ + patch.object(prereqs, "_run_logged") as logged: + os.environ.pop("CODE4ME_E2E_BROWSER_PYTHON", None) + self.assertEqual(prereqs.MISSING, prereqs.check_browser_python(False).status) + logged.assert_not_called() + + +class AgentsTest(unittest.TestCase): + def test_missing_goose_is_blocked_with_the_install_hint(self): + blocked = real_agents.ProbeResult.blocked("goose", real_agents.AgentProbeReason.MISSING_BINARY, + "no goose executable found") + with patch.object(prereqs.real_agents, "probe_identity", return_value=blocked): + check = prereqs.check_goose(True) + self.assertEqual(prereqs.BLOCKED, check.status) + self.assertIn("CODE4ME_E2E_GOOSE_EXECUTABLE", check.remediation) + + def _adapter(self, root: Path, *, built=True, installed=True, binary=True) -> Path: + adapter = root / "codex-acp" + (adapter / "src").mkdir(parents=True) + (adapter / "src/index.ts").write_text("export {}\n") + (adapter / "package.json").write_text("{}") + if installed: + (adapter / "node_modules").mkdir() + if binary: + native = adapter / "node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin/bin" + native.mkdir(parents=True) + (native / "codex").write_text("binary") + if built: + (adapter / "dist").mkdir() + (adapter / "dist/index.js").write_text("// built\n") + later = (adapter / "src/index.ts").stat().st_mtime + 10 + os.utime(adapter / "dist/index.js", (later, later)) + return adapter + + def test_vendored_adapter_is_wrapped_and_returned(self): + with tempfile.TemporaryDirectory() as tmp, \ + patch.dict(os.environ, {}, clear=False), \ + patch.object(prereqs, "AGENT_BIN_DIR", Path(tmp) / "bin"), \ + patch.object(prereqs.shutil, "which", side_effect=lambda name: f"/bin/{name}"): + os.environ.pop("CODE4ME_E2E_CODEX_EXECUTABLE", None) + adapter = self._adapter(Path(tmp)) + executables = {} + with patch.object(prereqs, "vendored_codex_dir", return_value=adapter), \ + patch.object(prereqs, "_run_logged") as logged: + check = prereqs.check_codex_acp(True, executables=executables) + logged.assert_not_called() + wrapper = Path(executables["codex"]) + self.assertEqual(prereqs.READY, check.status) + self.assertIn(str(adapter / "dist/index.js"), wrapper.read_text()) + self.assertTrue(os.access(wrapper, os.X_OK)) + + def test_check_mode_never_writes_the_launcher(self): + with tempfile.TemporaryDirectory() as tmp, \ + patch.dict(os.environ, {}, clear=False), \ + patch.object(prereqs, "AGENT_BIN_DIR", Path(tmp) / "bin"), \ + patch.object(prereqs.shutil, "which", side_effect=lambda name: f"/bin/{name}"): + os.environ.pop("CODE4ME_E2E_CODEX_EXECUTABLE", None) + adapter = self._adapter(Path(tmp)) + with patch.object(prereqs, "vendored_codex_dir", return_value=adapter): + check = prereqs.check_codex_acp(False, executables={}) + self.assertEqual(prereqs.MISSING, check.status) + self.assertFalse((Path(tmp) / "bin").exists()) + prereqs.check_codex_acp(True, executables={}) + executables = {} + self.assertEqual(prereqs.READY, prereqs.check_codex_acp(False, executables=executables).status) + self.assertTrue(executables["codex"].endswith("codex-acp")) + + def test_unbuilt_adapter_is_installed_and_built_only_when_provisioning(self): + with tempfile.TemporaryDirectory() as tmp, \ + patch.dict(os.environ, {}, clear=False), \ + patch.object(prereqs, "AGENT_BIN_DIR", Path(tmp) / "bin"), \ + patch.object(prereqs.shutil, "which", side_effect=lambda name: f"/bin/{name}"), \ + contextlib.redirect_stderr(io.StringIO()): + os.environ.pop("CODE4ME_E2E_CODEX_EXECUTABLE", None) + adapter = self._adapter(Path(tmp), built=False, installed=False, binary=False) + + def build(command, log_name, **_kwargs): + if command[1:] == ["ci"]: + native = adapter / "node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin/bin" + native.mkdir(parents=True) + (native / "codex").write_text("binary") + if command[1:] == ["run", "build"]: + (adapter / "dist").mkdir(exist_ok=True) + (adapter / "dist/index.js").write_text("// built\n") + return 0, Path("/tmp/log") + + with patch.object(prereqs, "vendored_codex_dir", return_value=adapter), \ + patch.object(prereqs, "_run_logged", side_effect=build) as logged: + self.assertEqual(prereqs.MISSING, prereqs.check_codex_acp(False, executables={}).status) + logged.assert_not_called() + check = prereqs.check_codex_acp(True, executables={}) + self.assertEqual(prereqs.PROVISIONED, check.status) + self.assertEqual([["/bin/npm", "ci"], ["/bin/npm", "run", "build"]], + [call.args[0] for call in logged.call_args_list]) + + def test_quarantined_native_codex_binary_is_blocked(self): + with tempfile.TemporaryDirectory() as tmp, \ + patch.dict(os.environ, {}, clear=False), \ + patch.object(prereqs, "AGENT_BIN_DIR", Path(tmp) / "bin"), \ + patch.object(prereqs.shutil, "which", side_effect=lambda name: f"/bin/{name}"): + os.environ.pop("CODE4ME_E2E_CODEX_EXECUTABLE", None) + adapter = self._adapter(Path(tmp), binary=False) + with patch.object(prereqs, "vendored_codex_dir", return_value=adapter): + check = prereqs.check_codex_acp(True, executables={}) + self.assertEqual(prereqs.BLOCKED, check.status) + self.assertIn("XProtect", check.detail) + + +class AgentProbeCliTest(unittest.TestCase): + def test_an_unbuilt_vendored_adapter_blocks_instead_of_falling_back_to_path(self): + missing = prereqs.Check("codex_acp", prereqs.MISSING, "the vendored adapter needs npm run build") + out = io.StringIO() + with tempfile.TemporaryDirectory() as tmp, \ + patch.dict(os.environ, {}, clear=False), \ + patch.object(cli.prereqs, "check_codex_acp", return_value=missing) as check, \ + patch.object(cli.real_agents, "probe_identity") as identity, \ + contextlib.redirect_stdout(out): + os.environ.pop("CODE4ME_E2E_CODEX_EXECUTABLE", None) + rc = cli.main(["agent-probe", "--framework", "codex", "--detect-only", "--json", + "--run-dir", tmp]) + self.assertEqual(1, rc) + identity.assert_not_called() + self.assertFalse(check.call_args.args[0], "the probe must never provision") + self.assertFalse(check.call_args.kwargs["write_wrapper"], "--detect-only must not write the launcher") + payload = json.loads(out.getvalue()) + self.assertEqual(("BLOCKED", "missing_binary"), (payload["status"], payload["reason"])) + self.assertIn("setup --layer agents", payload["detail"]) + + +class PrepareTest(unittest.TestCase): + def test_layers_share_checks_and_blockers_name_their_remediation(self): + seen = [] + + def fake(name, status): + def check(provision, **_kwargs): + seen.append(name) + return prereqs.Check(name, status, f"{name} detail", f"fix {name}") + return check + + fakes = {name: fake(name, prereqs.READY) for name in prereqs.CHECKS} + fakes["goose"] = fake("goose", prereqs.BLOCKED) + with patch.dict(prereqs.CHECKS, fakes): + result = prereqs.prepare(load_scenario(), ("backend", "agents")) + self.assertEqual(len(seen), len(set(seen)), "a shared check must run once") + self.assertFalse(result.ready) + self.assertEqual([], result.blocking("backend")) + self.assertEqual("goose: fix goose", result.blocked_reason("agents")) + self.assertEqual({"backend": "READY", "agents": "BLOCKED"}, result.to_dict()["layers"]) + + def test_an_unusable_daemon_blocks_dependent_checks_without_running_them(self): + ran = [] + + def fake(name, status): + def check(provision, **_kwargs): + ran.append(name) + return prereqs.Check(name, status, name, "start docker" if name == "docker" else "") + return check + + fakes = {name: fake(name, prereqs.READY) for name in prereqs.CHECKS} + fakes["docker"] = fake("docker", prereqs.BLOCKED) + with patch.dict(prereqs.CHECKS, fakes): + result = prereqs.prepare(load_scenario(), ("backend",)) + self.assertNotIn("backend_image", ran) + self.assertEqual(prereqs.BLOCKED, result.checks["backend_image"].status) + self.assertEqual("start docker", result.checks["backend_image"].remediation) + + def test_a_crashing_check_is_blocked_not_raised(self): + def boom(provision, **_kwargs): + raise RuntimeError("kaboom") + + fakes = {name: (lambda provision, name=name, **_k: prereqs.Check(name, prereqs.READY)) for name in prereqs.CHECKS} + fakes["node"] = boom + with patch.dict(prereqs.CHECKS, fakes): + result = prereqs.prepare(load_scenario(), ("agents",)) + self.assertEqual(prereqs.BLOCKED, result.checks["node"].status) + self.assertIn("kaboom", result.checks["node"].detail) + + def test_setup_cli_reports_json_and_exit_status(self): + blocked = prereqs.Prerequisites(("agents",), {"goose": prereqs.Check("goose", prereqs.BLOCKED, "absent", "install")}) + ready = prereqs.Prerequisites(("agents",), {"goose": prereqs.Check("goose", prereqs.READY, "ok")}) + for result, expected in ((blocked, 1), (ready, 0)): + out = io.StringIO() + with patch.object(cli.prereqs, "prepare", return_value=result) as prepare, \ + patch.object(cli.stack, "E2E_DIR", Path(tempfile.mkdtemp())), \ + contextlib.redirect_stdout(out): + rc = cli.main(["setup", "--layer", "agents", "--check", "--json"]) + self.assertEqual(expected, rc) + self.assertFalse(prepare.call_args.kwargs["provision"]) + self.assertEqual(result.ready, json.loads(out.getvalue())["ready"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e/tests/test_real_agents.py b/e2e/tests/test_real_agents.py new file mode 100644 index 0000000..653daa9 --- /dev/null +++ b/e2e/tests/test_real_agents.py @@ -0,0 +1,764 @@ +"""Regression tests for real-agent (BYOA) setup and scoped ACP evidence. + +These tests never launch a real Goose/Codex agent, Compose, an IDE, a browser or +a network provider. They use temporary directories, temporary fake executables +that speak a recorded ACP transcript over stdio, and the server's own normalizer +and profile/release validation modules to pin the harness contracts: + +* discovery order (explicit override, env override, declared command, PATH, + known per-user install location) and ``missing_binary`` reporting; +* executable identity (path, size, sha256, bounded ``--version`` capture) and + ``ProbeResult`` typing; +* the Codex ACP guard in :func:`code4me_e2e.real_agents.plan_argv`; +* typed BLOCKED reasons from recorded transcripts (auth, quota, protocol); +* scoping of :func:`code4me_e2e.acp.evaluate_telemetry` to this emitter and the + required canonical event counts (checked against the real + ``research.telemetry.normalization.generic_acp`` normalizer); +* workflow scenarios refusing Goose/Codex arms (they are BYOA; the agents + layer covers them); +* the stub provider's monotonic receipt delta; +* registered-entry identity failures (empty/missing command); +* the shared-lock bypass being limited to ``agent-probe``. +""" + +from __future__ import annotations + +import contextlib +import fcntl +import hashlib +import io +import json +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from code4me_e2e import acp, cli, real_agents, stack +from code4me_e2e.acp_probe import run_probe +from code4me_e2e.config import AgentProbeReason, ScenarioError, load_scenario +from code4me_e2e.stub_provider import StubProvider + +# The server's own contracts are verified where the server package is +# importable (the harness's documented venv, which the run's verification +# command uses). A bare interpreter that runs only the harness unit tests +# cannot import ``research.*``; those checks are then reported as skipped with +# this reason, never as a pass. +_SERVER_CONTRACT_SKIP = "the editable server package (research.*) is not installed" +try: + from research.telemetry.normalization.generic_acp import GenericAcpNormalizer + + _SERVER_CONTRACT_AVAILABLE = True +except ImportError: + _SERVER_CONTRACT_AVAILABLE = False + +_FAKE_ANSWER = "hello from fake agent" + +#: A recorded ACP transcript. The fake only ever exists in a temp directory and +#: is never a real agent: each mode drives one probe outcome. +_FAKE_AGENT_TEMPLATE = '''#!__PYTHON__ +"""Recorded ACP transcript (test fixture; not a real agent).""" +import json +import sys + +MODE = "__MODE__" +FRAMEWORK = "__FRAMEWORK__" +ANSWER = "__ANSWER__" + +if "--version" in sys.argv: + print("fake-%s 9.9.9" % FRAMEWORK) + raise SystemExit(0) + + +def send(payload): + sys.stdout.write(json.dumps(payload) + "\\n") + sys.stdout.flush() + + +for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + message = json.loads(line) + except ValueError: + continue + method = message.get("method") + if method == "initialize": + if MODE == "auth": + send({"jsonrpc": "2.0", "id": message["id"], + "error": {"code": 401, "message": "401 Unauthorized: not logged in"}}) + elif MODE == "quota": + send({"jsonrpc": "2.0", "id": message["id"], + "error": {"code": -32000, + "message": "429 Too Many Requests: usage limit reached"}}) + elif MODE == "protocol": + send({"jsonrpc": "2.0", "id": message["id"], + "result": {"protocolVersion": 0}}) + else: + send({"jsonrpc": "2.0", "id": message["id"], + "result": {"protocolVersion": 1, + "agentCapabilities": {"promptCapabilities": {}}}}) + elif method == "session/new": + send({"jsonrpc": "2.0", "id": message["id"], + "result": {"sessionId": "fake-session-1"}}) + elif method == "session/prompt": + send({"jsonrpc": "2.0", "method": "session/update", + "params": {"update": {"sessionUpdate": "agent_message_chunk", + "content": {"type": "text", "text": ANSWER}}}}) + send({"jsonrpc": "2.0", "id": message["id"], + "result": {"stopReason": "end_turn"}}) +''' + + +def _make_executable(directory, name: str, body: str) -> Path: + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + path = directory / name + path.write_text(body, encoding="utf-8") + path.chmod(0o755) + return path + + +def _version_script(framework: str) -> str: + return "#!/bin/sh\necho fake-%s 9.9.9\n" % framework + + +def _fake_agent_script(mode: str, framework: str) -> str: + return ( + _FAKE_AGENT_TEMPLATE.replace("__PYTHON__", sys.executable) + .replace("__MODE__", mode) + .replace("__FRAMEWORK__", framework) + .replace("__ANSWER__", _FAKE_ANSWER) + ) + + +def _write_registry(run_dir, entries) -> Path: + registry = Path(run_dir) / "ide-home" / ".jetbrains" / "acp.json" + registry.parent.mkdir(parents=True, exist_ok=True) + registry.write_text(json.dumps({"agent_servers": entries}), encoding="utf-8") + return registry + + +class TempDirTest(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.root = Path(self._tmp.name) + + +# --------------------------------------------------------------------------- +# Discovery +# --------------------------------------------------------------------------- + + +class DetectionTest(TempDirTest): + def test_detection_order_override_env_command_path(self): + explicit = _make_executable(self.root / "a", "goose", _version_script("goose")) + env_exe = _make_executable(self.root / "b", "goose", _version_script("goose")) + command_exe = _make_executable(self.root / "c", "goose-cmd", _version_script("goose")) + path_exe = _make_executable(self.root / "d", "goose", _version_script("goose")) + override_env = {real_agents.ENV_OVERRIDES["goose"]: str(env_exe)} + + found = real_agents.detect( + "goose", executable=str(explicit), command=str(command_exe), + env=override_env, path_env=str(self.root / "d"), + ) + self.assertEqual(("override", str(explicit)), (found.source, found.path)) + + found = real_agents.detect( + "goose", command=str(command_exe), env=override_env, + path_env=str(self.root / "d"), + ) + self.assertEqual(("env", str(env_exe)), (found.source, found.path)) + + found = real_agents.detect( + "goose", command=str(command_exe), env={}, path_env=str(self.root / "d") + ) + self.assertEqual(("command", str(command_exe)), (found.source, found.path)) + + found = real_agents.detect("goose", env={}, path_env=str(self.root / "d")) + self.assertEqual(("path", str(path_exe)), (found.source, found.path)) + + def test_missing_binary_is_none_and_never_searches_the_developer_home(self): + empty = self.root / "empty" + empty.mkdir() + with patch.object(real_agents, "known_locations", return_value=[]): + self.assertIsNone(real_agents.detect("goose", env={}, path_env=str(empty))) + + def test_known_location_fallback_matches_a_plugin_location(self): + location = self.root / "fake-home" / ".local" / "bin" + exe = _make_executable(location, "goose", _version_script("goose")) + empty = self.root / "empty" + empty.mkdir() + with patch.object(real_agents, "known_locations", return_value=[location]): + found = real_agents.detect("goose", env={}, path_env=str(empty)) + self.assertEqual(("known_location", str(exe)), (found.source, found.path)) + + def test_non_executable_override_does_not_silently_launch_another_agent(self): + not_executable = self.root / "bad" / "goose" + not_executable.parent.mkdir(parents=True) + not_executable.write_text("not executable", encoding="utf-8") + path_dir = self.root / "d" + _make_executable(path_dir, "goose", _version_script("goose")) # must not be chosen + found = real_agents.detect( + "goose", executable=str(not_executable), env={}, path_env=str(path_dir) + ) + self.assertIsNone(found) + + def test_declared_package_name_falls_back_to_its_own_executable(self): + package_dir = self.root / "pkg" + exe = _make_executable(package_dir, "custom-agent", _version_script("custom-agent")) + self.assertEqual(("custom-agent",), real_agents.candidate_names("goose", "custom-agent")) + found = real_agents.detect( + "goose", package="custom-agent", env={}, path_env=str(package_dir) + ) + self.assertEqual(("path", str(exe)), (found.source, found.path)) + + +# --------------------------------------------------------------------------- +# Identity and ProbeResult typing +# --------------------------------------------------------------------------- + + +class IdentityTest(TempDirTest): + def test_identity_records_path_size_sha256_and_version(self): + exe = _make_executable(self.root, "goose", _version_script("goose")) + identity = real_agents.executable_identity(exe, framework="goose", source="override") + self.assertEqual(str(exe.resolve()), identity.path) + self.assertEqual(exe.stat().st_size, identity.size) + self.assertEqual(hashlib.sha256(exe.read_bytes()).hexdigest(), identity.sha256) + self.assertEqual("fake-goose 9.9.9", identity.version) + self.assertEqual("override", identity.source) + self.assertEqual( + { + "framework": "goose", + "path": identity.path, + "size": identity.size, + "sha256": identity.sha256, + "version": identity.version, + "source": "override", + }, + identity.to_dict(), + ) + + def test_a_failing_version_flag_records_no_version(self): + exe = _make_executable(self.root, "proxy", "#!/bin/sh\necho 'usage: proxy --agent-cmd ...' >&2\nexit 2\n") + self.assertIsNone(real_agents.capture_version(exe)) + + def test_version_capture_is_bounded_when_the_fake_ignores_the_flag(self): + exe = _make_executable(self.root, "hanger", "#!/bin/sh\nsleep 30\n") + started = time.monotonic() + self.assertIsNone(real_agents.capture_version(exe, timeout=0.5)) + self.assertLess(time.monotonic() - started, 10) + + def test_reason_vocabulary_and_typed_probe_result(self): + self.assertEqual( + {"missing_binary", "missing_auth", "quota", "protocol"}, + {reason.value for reason in AgentProbeReason}, + ) + blocked = real_agents.ProbeResult.blocked( + "goose", AgentProbeReason.MISSING_AUTH, "no credential" + ) + self.assertEqual("BLOCKED", blocked.status) + self.assertFalse(blocked.passed) + self.assertEqual("missing_auth", blocked.reason) + self.assertIsInstance(blocked.reason, str) + self.assertNotIn("identity", blocked.to_dict()) + + identity = real_agents.AgentIdentity("goose", "/tmp/goose", 3, "a" * 64, "v", "override") + passed = real_agents.ProbeResult.ok("goose", identity) + self.assertTrue(passed.passed) + self.assertEqual(identity.to_dict(), passed.to_dict()["identity"]) + + def test_probe_identity_reports_missing_binary_with_the_override_variable(self): + empty = self.root / "empty" + empty.mkdir() + with patch.object(real_agents, "known_locations", return_value=[]): + result = real_agents.probe_identity("goose", env={}, path_env=str(empty)) + self.assertEqual("BLOCKED", result.status) + self.assertEqual("missing_binary", result.reason) + self.assertIn(real_agents.ENV_OVERRIDES["goose"], result.detail) + self.assertEqual("goose", result.protocol["searched"]) + self.assertIsNone(result.identity) + + def test_build_env_pins_an_isolated_home_and_strips_ambient_credentials(self): + home = self.root / "home" + home.mkdir() + env = real_agents.build_env( + home, + base_env={ + "PATH": "/bin", + "FOO": "bar", + "OPENAI_API_KEY": "sk-secret", + "CODE4ME_AGENT": "x", + "GOOSE_MODEL": "y", + }, + ) + self.assertEqual(str(home), env["HOME"]) + self.assertEqual(str(home / ".codex"), env["CODEX_HOME"]) + self.assertEqual("/bin", env["PATH"]) + self.assertEqual("bar", env["FOO"]) + for stripped in ("OPENAI_API_KEY", "CODE4ME_AGENT", "GOOSE_MODEL"): + self.assertNotIn(stripped, env) + + def test_isolated_home_is_created_under_the_run_directory(self): + home = real_agents.isolated_home(self.root / "run", "goose") + self.assertEqual(self.root / "run" / "agent-home" / "goose", home) + self.assertTrue(home.is_dir()) + override = real_agents.isolated_home(self.root / "run", "goose", str(self.root / "elsewhere")) + self.assertEqual(self.root / "elsewhere", override) + + def test_failure_text_redacts_bearer_and_keyed_tokens(self): + secret = "sk-" + "a" * 24 + text = real_agents.redact_failure_text( + "initialize failed: Authorization: Bearer %s 401 unauthorized" % secret + ) + self.assertNotIn(secret, text) + self.assertIn("", text) + self.assertIn("401 unauthorized", text) + + +# --------------------------------------------------------------------------- +# Launch plan +# --------------------------------------------------------------------------- + + +class PlanArgvTest(TempDirTest): + def test_goose_plan_appends_acp_and_honors_a_configured_argv(self): + identity = real_agents.AgentIdentity("goose", str(self.root / "goose"), 1, "a" * 64, "v", "override") + plan = real_agents.plan_argv("goose", identity) + self.assertFalse(plan.blocked) + self.assertEqual([identity.path, "acp"], plan.argv) + configured = real_agents.plan_argv("goose", identity, ["--debug", "acp"]) + self.assertFalse(configured.blocked) + self.assertEqual([identity.path, "--debug", "acp"], configured.argv) + + def test_codex_guard_blocks_the_bare_cli_but_allows_codex_acp(self): + bare = real_agents.AgentIdentity("codex", str(self.root / "codex"), 1, "a" * 64, "v", "override") + blocked = real_agents.plan_argv("codex", bare) + self.assertTrue(blocked.blocked) + self.assertEqual(AgentProbeReason.PROTOCOL, blocked.blocked_reason) + self.assertIn("codex-acp", blocked.blocked_detail) + + adapter = real_agents.AgentIdentity("codex", str(self.root / "codex-acp"), 1, "a" * 64, "v", "override") + self.assertEqual([adapter.path], real_agents.plan_argv("codex", adapter).argv) + + configured = real_agents.plan_argv("codex", bare, ["--experimental-acp"]) + self.assertFalse(configured.blocked) + self.assertEqual([bare.path, "--experimental-acp"], configured.argv) + + def test_empty_executable_entry_is_a_missing_binary_block(self): + empty = real_agents.AgentIdentity("goose", "", 0, "", None, "override") + plan = real_agents.plan_argv("goose", empty) + self.assertTrue(plan.blocked) + self.assertEqual(AgentProbeReason.MISSING_BINARY, plan.blocked_reason) + self.assertEqual([], plan.argv) + + +# --------------------------------------------------------------------------- +# Real ACP probe against recorded transcripts +# --------------------------------------------------------------------------- + + +class RecordedProbeTest(TempDirTest): + def _fake(self, name: str, mode: str, framework: str = "goose") -> Path: + return _make_executable(self.root / "bin", name, _fake_agent_script(mode, framework)) + + def test_recorded_transcripts_classify_blocked_reasons(self): + for mode, expected in ( + ("auth", "missing_auth"), + ("quota", "quota"), + ("protocol", "protocol"), + ): + with self.subTest(mode=mode): + exe = self._fake("goose", mode) + result = run_probe( + "goose", run_dir=self.root / ("run-" + mode), + executable=str(exe), timeout=10, + ) + self.assertEqual("BLOCKED", result.status) + self.assertFalse(result.passed) + self.assertEqual(expected, result.reason) + self.assertIsNotNone(result.identity) + self.assertEqual("fake-goose 9.9.9", result.identity.version) + self.assertEqual(["initialize"], result.protocol["methods"]) + self.assertIsInstance(result.detail, str) + + def test_successful_fake_session_is_a_pass_with_an_isolated_home(self): + exe = self._fake("goose", "ok") + run_dir = self.root / "run-ok" + result = run_probe("goose", run_dir=run_dir, executable=str(exe), timeout=10) + self.assertEqual("PASS", result.status) + self.assertEqual( + ["initialize", "session/new", "session/prompt"], + [check["check"] for check in result.checks], + ) + self.assertEqual(1, result.checks[0]["protocol_version"]) + self.assertTrue(result.checks[1]["session_id_present"]) + self.assertEqual("end_turn", result.checks[2]["stop_reason"]) + self.assertEqual(len(_FAKE_ANSWER), result.checks[2]["assistant_message_chars"]) + self.assertEqual(["initialize", "session/new", "session/prompt"], result.protocol["methods"]) + home = run_dir / "agent-home" / "goose" + self.assertEqual(str(home), result.protocol["home"]) + self.assertTrue(home.is_dir()) + self.assertNotEqual(str(Path.home()), result.protocol["home"]) + self.assertTrue((run_dir / "agent-goose.stderr.log").is_file()) + + def test_codex_guard_blocks_the_bare_cli_before_any_launch(self): + exe = self._fake("codex", "ok", framework="codex") + run_dir = self.root / "run-codex" + result = run_probe("codex", run_dir=run_dir, executable=str(exe), timeout=10) + self.assertEqual("BLOCKED", result.status) + self.assertEqual("protocol", result.reason) + self.assertIn("codex-acp", result.detail) + self.assertFalse((run_dir / "agent-home").exists()) + + def test_codex_acp_adapter_is_launched_without_a_subcommand(self): + exe = self._fake("codex-acp", "ok", framework="codex") + result = run_probe( + "codex", run_dir=self.root / "run-codex-acp", executable=str(exe), timeout=10 + ) + self.assertEqual("PASS", result.status) + self.assertEqual([str(exe.resolve())], result.protocol["argv"]) + + +# --------------------------------------------------------------------------- +# Scoped telemetry +# --------------------------------------------------------------------------- + + +class TelemetryScopeTest(unittest.TestCase): + EMITTER = "acp-proxy:e2e-test" + + @staticmethod + def _rows(emitter, counts): + return [(emitter, "acp", event_type, count) for event_type, count in counts.items()] + + @unittest.skipUnless(_SERVER_CONTRACT_AVAILABLE, _SERVER_CONTRACT_SKIP) + def test_required_counts_match_the_server_normalizer(self): + """The required constants are derived from the real normalizer.""" + normalizer = GenericAcpNormalizer() + frames = [ + {"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": 1}}, + {"jsonrpc": "2.0", "id": 2, "method": "session/new", "params": {"cwd": "/tmp"}}, + # The chat start is taken from the agent's answer, which names the chat. + {"jsonrpc": "2.0", "id": 2, "result": {"sessionId": "s1"}}, + {"jsonrpc": "2.0", "id": 3, "method": "session/prompt", "params": {"sessionId": "s1"}}, + { + "jsonrpc": "2.0", + "method": "session/update", + "params": { + "update": { + "sessionUpdate": "agent_message_chunk", + "content": {"type": "text", "text": "hi"}, + } + }, + }, + {"jsonrpc": "2.0", "id": 3, "result": {"stopReason": "end_turn"}}, + ] + counts = {} + for frame in frames: + for candidate in normalizer.normalize(frame).candidates: + counts[candidate.event_type.value] = counts.get(candidate.event_type.value, 0) + 1 + self.assertEqual(dict(acp.REQUIRED_ACP_EVENT_COUNTS), counts) + self.assertEqual( + { + "initialize": "interaction.started", + "session/new": "interaction.started", + "session/prompt": "agent.message.started", + }, + dict(acp.ACP_METHOD_EVENT_TYPES), + ) + + def test_scope_sql_bounds_rows_and_sanitizes_literals(self): + sql = acp.telemetry_scope_sql( + "11111111-1111-1111-1111-111111111111", + "acp-proxy:a'b", + "2026-01-01T00:00:00+00:00", + ) + self.assertIn("enrollment_id='11111111-1111-1111-1111-111111111111'", sql) + self.assertIn("occurred_at >=", sql) + self.assertIn("emitter_id LIKE 'ide:%'", sql) + self.assertIn("acp-proxy:ab", sql) + self.assertNotIn("a'b", sql) + with self.assertRaises(ValueError): + acp.telemetry_scope_sql("not-a-uuid", self.EMITTER, "2026-01-01T00:00:00+00:00") + + def test_ide_rows_are_session_scoped_but_not_time_windowed(self): + """The IDE emits its own events before the harness launches the proxy.""" + session = "22222222-2222-2222-2222-222222222222" + sql = acp.telemetry_scope_sql( + "11111111-1111-1111-1111-111111111111", + "acp-proxy:e2e", + "2026-01-01T00:00:00+00:00", + research_session_id=session, + ) + self.assertIn("(emitter_id='acp-proxy:e2e' AND occurred_at >= '2026-01-01T00:00:00+00:00')", sql) + self.assertIn("OR emitter_id LIKE 'ide:%')", sql) + # The session filter applies to both branches, the time window only to the proxy. + self.assertLess(sql.index(f"research_session_id='{session}'"), sql.index("AND ((")) + + def test_evaluate_telemetry_requires_this_emitters_counts_and_ide_source(self): + complete = self._rows(self.EMITTER, acp.REQUIRED_ACP_EVENT_COUNTS) + [ + ("ide:run-1", "ide", "interaction.started", 4), + ] + result = acp.evaluate_telemetry(complete, self.EMITTER) + self.assertEqual(["acp", "ide"], result["sources"]) + self.assertEqual(dict(acp.REQUIRED_ACP_EVENT_COUNTS), result["required_event_counts"]) + + # Counts split across rows accumulate. + split = [ + (self.EMITTER, "acp", "interaction.started", 1), + (self.EMITTER, "acp", "interaction.started", 1), + (self.EMITTER, "acp", "agent.message.started", 2), + (self.EMITTER, "acp", "agent.message.completed", 1), + ("ide:run-1", "ide", "interaction.started", 1), + ] + acp.evaluate_telemetry(split, self.EMITTER) + + # Another proxy's rows must not satisfy this emitter's requirement. + other = self._rows("acp-proxy:other", acp.REQUIRED_ACP_EVENT_COUNTS) + [ + ("ide:run-1", "ide", "interaction.started", 4), + ] + with self.assertRaises(AssertionError) as caught: + acp.evaluate_telemetry(other, self.EMITTER) + self.assertIn("missing_events", str(caught.exception)) + + # Without this run's IDE source the projection is incomplete. + with self.assertRaises(AssertionError) as caught: + acp.evaluate_telemetry(self._rows(self.EMITTER, acp.REQUIRED_ACP_EVENT_COUNTS), self.EMITTER) + self.assertIn("missing_sources", str(caught.exception)) + self.assertIn("ide", str(caught.exception)) + + def test_parse_telemetry_rows_skips_malformed_lines(self): + rows = acp.parse_telemetry_rows( + "ide:1|ide|interaction.started|3\nmalformed\nacp|acp|agent.message.started|nope\n" + ) + self.assertEqual([("ide:1", "ide", "interaction.started", 3)], rows) + + def test_scope_sql_requires_the_run_session_for_ide_rows(self): + session = "22222222-2222-2222-2222-222222222222" + sql = acp.telemetry_scope_sql( + "11111111-1111-1111-1111-111111111111", + "acp-proxy:e2e", + "2026-01-01T00:00:00+00:00", + research_session_id=session, + ) + self.assertIn(f"research_session_id='{session}'", sql) + self.assertIn("emitter_id LIKE 'ide:%'", sql) + # Without a session id there is no session filter at all (enrollment scope only). + unscoped = acp.telemetry_scope_sql( + "11111111-1111-1111-1111-111111111111", + "acp-proxy:e2e", + "2026-01-01T00:00:00+00:00", + ) + self.assertNotIn("research_session_id", unscoped) + with self.assertRaises(ValueError): + acp.telemetry_scope_sql( + "11111111-1111-1111-1111-111111111111", + "acp-proxy:e2e", + "2026-01-01T00:00:00+00:00", + research_session_id="not-a-uuid", + ) + + +# --------------------------------------------------------------------------- +# Workflow scenarios and BYOA frameworks +# --------------------------------------------------------------------------- + + +class WorkflowFrameworkTest(unittest.TestCase): + def test_goose_and_codex_arms_are_refused_by_the_workflow_scenario(self): + for framework in ("goose", "codex"): + with self.subTest(framework=framework), self.assertRaises(ScenarioError) as caught: + load_scenario(overrides=[f"agent.framework_version={framework}"]) + self.assertEqual("FRAMEWORK_NOT_IN_WORKFLOW", caught.exception.code) + self.assertIn("--layer agents", str(caught.exception)) + + def test_probe_inputs_are_validated(self): + with self.assertRaises(ScenarioError) as caught: + load_scenario(overrides=['agent.agent_command_args=["acp", ""]']) + self.assertEqual("AGENT_COMMAND_ARGS_INVALID", caught.exception.code) + with self.assertRaises(ScenarioError): + load_scenario(overrides=['agent.executable=" "']) + scenario = load_scenario(overrides=['agent.agent_command_args=["--debug", "acp"]']) + self.assertEqual(["--debug", "acp"], scenario.agent.agent_command_args) + + +# --------------------------------------------------------------------------- +# Stub provider receipts +# --------------------------------------------------------------------------- + + +class StubReceiptTest(unittest.TestCase): + def test_request_receipts_are_monotonic_and_the_delta_is_per_turn(self): + stub = StubProvider() + self.assertEqual(0, stub.request_count()) + stub.record({"model": "m", "message_count": 1, "stream": False}) + first = stub.request_count() + stub.record({"model": "m", "message_count": 1, "stream": False}) + second = stub.request_count() + self.assertEqual((1, 2), (first, second)) + self.assertEqual([1, 2], [entry["receipt"] for entry in stub.requests]) + self.assertEqual(2, len(stub.requests_since(0))) + self.assertEqual(1, len(stub.requests_since(first))) + self.assertEqual([], stub.requests_since(second)) + + # The ACP exercise asserts exactly this delta for one first prompt: a + # receipt left by any earlier UI/prefix run cannot satisfy it. + before = stub.request_count() + stub.record({"model": "m", "message_count": 2, "stream": False}) + self.assertEqual(1, stub.request_count() - before) + + def test_completion_body_carries_the_deterministic_token_only(self): + stub = StubProvider(token="E2E_TEST_TOKEN") + body = stub.completion_body("e2e-stub-model") + self.assertIn("E2E_TEST_TOKEN", body["choices"][0]["message"]["content"]) + self.assertNotEqual(StubProvider().token, StubProvider().token) + + +# --------------------------------------------------------------------------- +# Registered ACP entry identity +# --------------------------------------------------------------------------- + + +class EntryIdentityTest(TempDirTest): + def _proxy(self) -> Path: + return _make_executable(self.root / "bin", "proxy", "#!/bin/sh\necho fake-proxy 1.0\n") + + def _agent(self) -> Path: + return _make_executable(self.root / "bin", "agent", "#!/bin/sh\necho fake-agent 1.0\n") + + def test_missing_registry_or_research_entry_fails_closed(self): + with self.assertRaises(RuntimeError): + acp.entry_identity(self.root) + _write_registry(self.root, {"Other Agent": {"command": "/bin/sh"}}) + with self.assertRaises(RuntimeError) as caught: + acp.entry_identity(self.root) + self.assertIn("Expected one research proxy registration", str(caught.exception)) + + def test_empty_or_missing_command_is_refused(self): + for entry in ({"args": []}, {"command": "", "args": []}, {"command": " ", "args": []}): + with self.subTest(entry=entry): + _write_registry(self.root, {"Code4Me Research Proxy": entry}) + with self.assertRaises(RuntimeError) as caught: + acp.entry_identity(self.root) + self.assertIn("empty command", str(caught.exception)) + + def test_nonexistent_command_is_refused(self): + _write_registry( + self.root, + {"Code4Me Research Proxy": {"command": str(self.root / "missing"), "args": []}}, + ) + with self.assertRaises(RuntimeError) as caught: + acp.entry_identity(self.root) + self.assertIn("does not exist", str(caught.exception)) + + def test_missing_agent_cmd_is_refused(self): + proxy = self._proxy() + _write_registry( + self.root, + {"Code4Me Research Proxy": {"command": str(proxy), "args": ["--emitter-id", "e"]}}, + ) + with self.assertRaises(RuntimeError) as caught: + acp.entry_identity(self.root) + self.assertIn("--agent-cmd", str(caught.exception)) + + def test_agent_digest_mismatch_is_refused(self): + proxy, agent = self._proxy(), self._agent() + _write_registry( + self.root, + { + "Code4Me Research Proxy": { + "command": str(proxy), + "args": ["--agent-digest", "deadbeef", "--agent-cmd", str(agent)], + } + }, + ) + with self.assertRaises(RuntimeError) as caught: + acp.entry_identity(self.root) + self.assertIn("does not match", str(caught.exception)) + + def test_identity_binds_the_proxy_and_the_agent_digest(self): + proxy, agent = self._proxy(), self._agent() + digest = hashlib.sha256(agent.read_bytes()).hexdigest() + _write_registry( + self.root, + { + "Code4Me Research Proxy": { + "command": str(proxy), + "args": ["--agent-digest", digest, "--agent-cmd", str(agent)], + "env": {"CODE4ME_AGENT_ADAPTER_ID": "adapter-1"}, + } + }, + ) + entry = acp.entry_identity(self.root) + self.assertEqual(digest, entry["agent"]["sha256"]) + self.assertTrue(entry["agent_digest_flag_matches"]) + self.assertEqual(["CODE4ME_AGENT_ADAPTER_ID"], entry["env_keys"]) + self.assertEqual("adapter-1", entry["adapter_id"]) + self.assertEqual(str(agent), entry["agent_argv"][0]) + self.assertEqual(str(agent.resolve()), entry["agent"]["path"]) + + def test_prefixed_agent_digest_matches_the_bare_hex(self): + proxy, agent = self._proxy(), self._agent() + digest = hashlib.sha256(agent.read_bytes()).hexdigest() + _write_registry( + self.root, + { + "Code4Me Research Proxy": { + "command": str(proxy), + "args": ["--agent-digest", "sha256:" + digest, "--agent-cmd", str(agent)], + } + }, + ) + entry = acp.entry_identity(self.root) + self.assertTrue(entry["agent_digest_flag_matches"]) + self.assertEqual(digest, entry["agent"]["sha256"]) + + def test_whitespace_agent_cmd_is_refused_as_empty(self): + proxy = self._proxy() + _write_registry( + self.root, + {"Code4Me Research Proxy": {"command": str(proxy), "args": ["--agent-cmd", " "]}}, + ) + with self.assertRaises(RuntimeError) as caught: + acp.entry_identity(self.root) + self.assertIn("empty --agent-cmd executable", str(caught.exception)) + + +# --------------------------------------------------------------------------- +# CLI lock bypass +# --------------------------------------------------------------------------- + + +class CliLockTest(TempDirTest): + def test_agent_probe_is_the_only_shared_lock_bypass(self): + cache = self.root / ".cache" + cache.mkdir() + lock_path = cache / "run.lock" + with patch.object(stack, "E2E_DIR", self.root), lock_path.open("w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + probe_rc = cli.main( + [ + "agent-probe", + "--framework", + "goose", + "--executable", + sys.executable, + "--detect-only", + "--json", + ] + ) + blocked_rc = cli.main(["stack", "status"]) + self.assertEqual(0, probe_rc, stdout.getvalue()) + self.assertEqual(2, blocked_rc) + self.assertIn("Another e2e command is running", stderr.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e/tests/test_suite_coverage.py b/e2e/tests/test_suite_coverage.py new file mode 100644 index 0000000..1941311 --- /dev/null +++ b/e2e/tests/test_suite_coverage.py @@ -0,0 +1,421 @@ +"""Regression tests: complete `all` gating, browser evidence inventory, isolation.""" +import contextlib +import io +import json +import re +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from _support import isolate_prerequisites +from code4me_e2e import browser, suite, ui, workflow +from code4me_e2e.config import load_scenario +from code4me_e2e.steps import StepResult + +E2E_DIR = Path(__file__).resolve().parents[1] +SCENARIOS_PATH = E2E_DIR / "browser" / "scenarios.py" + +PERSISTED_CONTEXT = { + "study_id": "study-1", + "edited_name": "Browser study (edited)", + "edited_description": "Edited description", + "clone_study_id": "study-2", + "clone_name": "Browser study (edited) (copy)", + "profile_id": "profile-1", + "enrollment_id": "enrollment-1", + "assignment_id": "assignment-1", +} + + +def _persisted_responses(): + return { + "/api/research/studies/study-1": {"status": 200, "payload": {"study": { + "name": "Browser study (edited)", "description": "Edited description", + "consent_locked_at": "2026-01-01T00:00:00Z", "research_status": "STUDY_STOPPED", + "enrollment_count": 1, "active_enrollment_count": 0, "assignment_count": 1, + }}}, + "/api/research/studies/study-2": {"status": 200, "payload": {"study": { + "name": "Browser study (edited) (copy)", "description": "Edited description", + "research_status": "DRAFT", "profile_selections": [{"profile_id": "profile-1"}], + "enrollment_count": 0, "assignment_count": 0, + "lifecycle_capabilities": {"joinable": True}, + }}}, + "coverage": {"status": 200, "payload": { + "participant_count": 1, + "participants": [{ + "enrollment_id": "enrollment-1", "status": "STUDY_STOPPED", + "assignment": {"assignment_id": "assignment-1", + "agent_profile_id": "profile-1", + "status": "STUDY_STOPPED"}, + }], + }}, + } + + +def _fake_api(responses): + def fake(base_url, path, cookies, timeout=20.0): + if "/operations/participants/coverage" in path: + return responses["coverage"] + for key, response in responses.items(): + if key.startswith("/api/research/studies/") and path.endswith(key.rsplit("/", 1)[-1]): + return response + raise AssertionError(f"unexpected read: {path}") + return fake + + +def _record_ui(run_dir, scenario, status, rc): + ui.record_ui_step(Path(run_dir), scenario, StepResult("ui_test", status, 1, {}), {}) + return rc + + +def _record_plugin(run_dir, scenario, status, rc): + workflow._record_plugin_step(Path(run_dir), scenario, StepResult("plugin_test", status, 1, {})) + return rc + + +def _record_backend(run_dir, scenario, rc): + workflow._record_plugin_step(Path(run_dir), scenario, StepResult("verify", "PASS", 1, {})) + return rc + + +def _browser_side_effect(status, rc): + def _run(scenario, run_path, **kwargs): + workflow._record_plugin_step(Path(run_path), scenario, StepResult("browser_test", status, 1, {})) + return rc + return _run + + +def _record_plugin_step_for_agents(scenario, path, **_kwargs): + workflow._record_plugin_step(Path(path), scenario, StepResult("agents_test", "PASS", 1)) + return 0 + + +class LayerGateTest(unittest.TestCase): + def setUp(self): + isolate_prerequisites(self) + patcher = patch.object(suite.native_agents, "run", side_effect=_record_plugin_step_for_agents) + patcher.start() + self.addCleanup(patcher.stop) + + def test_layer_plan_keeps_the_dependent_chain_and_adds_browser(self): + self.assertEqual(("ui", "plugin", "backend", "agents", "browser"), suite.plan_layers("all")) + self.assertEqual(("plugin", "backend"), suite.plan_layers("plugin")) + self.assertEqual(("backend",), suite.plan_layers("backend")) + self.assertEqual(("browser",), suite.plan_layers("browser")) + + def test_ui_failure_cannot_skip_the_independent_browser_layer(self): + scenario = load_scenario() + with tempfile.TemporaryDirectory() as directory, \ + patch.object(ui, "run_ui_test", + side_effect=lambda s, **kw: _record_ui(kw["run_dir"], s, "FAIL", 1)), \ + patch.object(workflow, "run_plugin_test") as plugin, \ + patch.object(workflow, "run_workflow") as backend, \ + patch.object(browser, "run", side_effect=_browser_side_effect("PASS", 0)), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + rc = suite.run(scenario, run_dir=directory) + built = json.loads((Path(directory) / "report.json").read_text()) + self.assertEqual(1, rc) + plugin.assert_not_called() + backend.assert_not_called() + self.assertEqual([item["layer"] for item in built["layers"]], + ["ui", "plugin", "backend", "agents", "browser"]) + layers = {item["layer"]: item for item in built["layers"]} + self.assertEqual("FAIL", layers["ui"]["status"]) + self.assertEqual("NOT_ATTEMPTED", layers["plugin"]["status"]) + self.assertEqual("NOT_ATTEMPTED", layers["backend"]["status"]) + self.assertFalse(layers["plugin"]["attempted"]) + self.assertEqual("PASS", layers["browser"]["status"]) + self.assertTrue(layers["browser"]["attempted"]) + self.assertEqual("ui_test", built["failed_step"]) + + def test_browser_failure_fails_the_gate_and_keeps_every_layer_result(self): + scenario = load_scenario() + with tempfile.TemporaryDirectory() as directory, \ + patch.object(ui, "run_ui_test", + side_effect=lambda s, **kw: _record_ui(kw["run_dir"], s, "PASS", 0)), \ + patch.object(workflow, "run_plugin_test", + side_effect=lambda s, **kw: _record_plugin(kw["run_dir"], s, "PASS", 0)), \ + patch.object(workflow, "run_workflow", + side_effect=lambda s, **kw: _record_backend(kw["run_dir"], s, 0)), \ + patch.object(browser, "run", side_effect=_browser_side_effect("FAIL", 1)), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + rc = suite.run(scenario, run_dir=directory) + built = json.loads((Path(directory) / "report.json").read_text()) + self.assertEqual(1, rc) + self.assertEqual([step["id"] for step in built["steps"]], + ["prerequisites", "ui_test", "plugin_test", "verify", "agents_test", "browser_test"]) + self.assertEqual("browser_test", built["failed_step"]) + self.assertTrue(all(item["attempted"] for item in built["layers"])) + + def test_blocked_browser_layer_is_not_a_pass(self): + scenario = load_scenario() + with tempfile.TemporaryDirectory() as directory, \ + patch.object(ui, "run_ui_test", + side_effect=lambda s, **kw: _record_ui(kw["run_dir"], s, "PASS", 0)), \ + patch.object(workflow, "run_plugin_test", + side_effect=lambda s, **kw: _record_plugin(kw["run_dir"], s, "PASS", 0)), \ + patch.object(workflow, "run_workflow", + side_effect=lambda s, **kw: _record_backend(kw["run_dir"], s, 0)), \ + patch.object(browser, "run", side_effect=_browser_side_effect("BLOCKED", 1)), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + rc = suite.run(scenario, run_dir=directory) + built = json.loads((Path(directory) / "report.json").read_text()) + self.assertEqual(1, rc) + layers = {item["layer"]: item for item in built["layers"]} + self.assertEqual("BLOCKED", layers["browser"]["status"]) + self.assertEqual("browser_test", built["blocked_by"]) + + def test_layer_outcome_reports_blocked_with_its_reason(self): + with tempfile.TemporaryDirectory() as directory: + run_path = Path(directory) + workflow._record_plugin_step( + run_path, load_scenario(), + StepResult("browser_test", "BLOCKED", 1, {"error": "no playwright"}, "install Playwright"), + ) + self.assertEqual( + ("BLOCKED", "install Playwright"), + suite.layer_outcome(run_path, step_id="browser_test", rc=1), + ) + self.assertEqual(("BLOCKED", "install Playwright"), suite.layer_outcome(run_path, step_id="browser_test", rc=0)) + + def test_browser_only_invocation_reports_just_its_own_layer(self): + with tempfile.TemporaryDirectory() as directory, \ + patch.object(browser, "run", side_effect=_browser_side_effect("PASS", 0)), \ + patch.object(suite.stack, "down"), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + rc = suite.run(load_scenario(), run_dir=directory, layer="browser") + built = json.loads((Path(directory) / "report.json").read_text()) + self.assertEqual(0, rc) + self.assertEqual( + [{"layer": "browser", "status": "PASS", "attempted": True, "reason": ""}], + built["layers"], + ) + + +class BlockedPrerequisiteIsolationTest(unittest.TestCase): + def test_a_missing_goose_blocks_only_the_agents_layer(self): + from _support import ready_prerequisites + from code4me_e2e import prereqs + from code4me_e2e import steps as step_module + + def only_goose_blocked(scenario, layers, *, provision=True): + result = ready_prerequisites(scenario, layers) + result.checks["goose"] = prereqs.Check("goose", prereqs.BLOCKED, "no goose", "install Goose") + return result + + def fake_ui(scenario, *, run_dir, keep_stack=False, **_kwargs): + # Like ui.run_ui_test: the real workflow prefix, then the ui step. + rc = workflow.run_workflow(scenario, only=",".join(workflow.PLUGIN_PREFIX_STEPS), + run_dir=run_dir, keep_stack=True) + return _record_ui(run_dir, scenario, "PASS" if rc == 0 else "FAIL", rc) + + passing = {step_id: (lambda ctx: {}) for step_id in step_module.STEP_ORDER} + with tempfile.TemporaryDirectory() as directory, \ + patch.object(suite.prereqs, "prepare", side_effect=only_goose_blocked), \ + patch.dict(workflow.STEPS, passing), \ + patch.object(workflow, "_own_stack", return_value=False), \ + patch.object(ui, "run_ui_test", side_effect=fake_ui), \ + patch.object(workflow, "run_plugin_test", + side_effect=lambda s, **kw: _record_plugin(kw["run_dir"], s, "PASS", 0)), \ + patch.object(suite.native_agents, "run") as agents, \ + patch.object(browser, "run", side_effect=_browser_side_effect("PASS", 0)), \ + patch.object(suite.report, "capture_backend_logs"), \ + patch.object(suite.stack, "down") as down, \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + rc = suite.run(load_scenario(), run_dir=directory) + built = json.loads((Path(directory) / "report.json").read_text()) + layers = {item["layer"]: item["status"] for item in built["layers"]} + self.assertEqual({"ui": "PASS", "plugin": "PASS", "backend": "PASS", "agents": "BLOCKED", + "browser": "PASS"}, layers) + self.assertEqual(1, rc, "a blocked layer still fails the gate") + agents.assert_not_called() + down.assert_not_called() + self.assertIn("install Goose", next(item for item in built["layers"] if item["layer"] == "agents")["reason"]) + + +class BrowserEvidenceTest(unittest.TestCase): + def _payload(self, *, ids=None, statuses=None, context=True): + steps = [ + {"id": step_id, "title": step_id, "status": (statuses or {}).get(step_id, "PASS"), "detail": ""} + for step_id in (ids if ids is not None else browser.EXPECTED_SCENARIO_IDS) + ] + payload = {"steps": steps} + if context: + payload["context"] = dict(PERSISTED_CONTEXT) + return payload + + def test_exact_inventory_passes_and_is_returned(self): + steps = browser.validate_scenario_results(self._payload()) + self.assertEqual(len(browser.EXPECTED_SCENARIO_IDS), len(steps)) + + def test_missing_duplicated_blocked_unexpected_and_failed_ids_cannot_pass(self): + without_clone = [step_id for step_id in browser.EXPECTED_SCENARIO_IDS if step_id != "C4"] + cases = { + "missing": (self._payload(ids=without_clone), "missing C4"), + "duplicated": (self._payload(ids=[*browser.EXPECTED_SCENARIO_IDS, "A1"]), "duplicated A1"), + "blocked": (self._payload(statuses={"C4": "BLOCKED"}), "not passing C4:BLOCKED"), + "failed": (self._payload(statuses={"B6": "FAIL"}), "not passing B6:FAIL"), + "unexpected": (self._payload(ids=[*browser.EXPECTED_SCENARIO_IDS, "Z9"]), "unexpected Z9"), + "empty": ({"steps": []}, "no steps array"), + } + for name, (payload, expected) in cases.items(): + with self.subTest(name=name), self.assertRaises(RuntimeError) as raised: + browser.validate_scenario_results(payload) + self.assertIn(expected, str(raised.exception)) + + def test_context_requires_identity_and_distinct_clone(self): + self.assertEqual(PERSISTED_CONTEXT, browser.validate_context(self._payload())) + for context in ({**PERSISTED_CONTEXT, "clone_study_id": ""}, + {**PERSISTED_CONTEXT, "clone_study_id": "study-1"}): + with self.subTest(context=context), self.assertRaises(RuntimeError): + browser.validate_context(self._payload() | {"context": context}) + + def test_secret_material_is_refused(self): + browser.assert_no_secret_material("clean evidence", secrets=["JOINCODE", ""]) + with self.assertRaises(RuntimeError) as raised: + browser.assert_no_secret_material('{"join_code": "JOINCODE"}', secrets=["JOINCODE"]) + self.assertIn("private material", str(raised.exception)) + + def test_persisted_outcomes_are_re_read_from_the_api_models(self): + state = {"cookies": {"researcher": {"auth_token": "token"}}} + with patch.object(browser, "_api_get", side_effect=_fake_api(_persisted_responses())): + checks = browser.verify_persisted_outcomes(load_scenario(), state, PERSISTED_CONTEXT) + self.assertTrue(all(checks.values())) + self.assertEqual( + {"edited_metadata_persisted", "stopped_state_retained", + "clone_identity_profiles_status", "enrollment_assignment_coverage"}, + set(checks), + ) + + def test_persisted_outcomes_fail_on_any_stale_read_model(self): + state = {"cookies": {"researcher": {"auth_token": "token"}}} + responses = _persisted_responses() + responses["/api/research/studies/study-2"]["payload"]["study"]["research_status"] = "ACTIVE" + with patch.object(browser, "_api_get", side_effect=_fake_api(responses)), \ + self.assertRaises(RuntimeError) as raised: + browser.verify_persisted_outcomes(load_scenario(), state, PERSISTED_CONTEXT) + self.assertIn("clone_identity_profiles_status", str(raised.exception)) + + def test_persisted_outcomes_require_a_researcher_session(self): + with self.assertRaises(RuntimeError): + browser.verify_persisted_outcomes(load_scenario(), {}, PERSISTED_CONTEXT) + + +class ScenarioSourceTest(unittest.TestCase): + def test_scenarios_declare_every_expected_id_exactly_once(self): + source = SCENARIOS_PATH.read_text(encoding="utf-8") + declared = re.findall(r'\brecord\(\s*"([A-Za-z0-9]+)"', source) + self.assertIn("harness", declared) + self.assertEqual( + sorted(browser.EXPECTED_SCENARIO_IDS), sorted(set(declared) - {"harness"}) + ) + self.assertEqual( + len(browser.EXPECTED_SCENARIO_IDS), len(set(browser.EXPECTED_SCENARIO_IDS)) + ) + blocked = re.findall(r'\brecord_blocked\(\s*"([A-Za-z0-9]+)"', source) + self.assertTrue(set(blocked) <= set(browser.EXPECTED_SCENARIO_IDS)) + self.assertIn("CODE4ME_E2E_BROWSER_EXPECTED_IDS", source) + self.assertNotIn("code={join_code}", source) + self.assertNotIn("prefilled={prefilled}", source) + + def test_runner_passes_the_inventory_to_the_child(self): + source = (E2E_DIR / "code4me_e2e" / "browser.py").read_text(encoding="utf-8") + self.assertIn("CODE4ME_E2E_BROWSER_EXPECTED_IDS", source) + self.assertIn("browser-state", source) + + +class ScenarioModuleTest(unittest.TestCase): + def test_scenario_module_records_ids_without_a_live_browser(self): + import importlib.util + import sys + from types import ModuleType + + fake_playwright = ModuleType("playwright.sync_api") + fake_playwright.sync_playwright = lambda: None + package = ModuleType("playwright") + with patch.dict(sys.modules, {"playwright": package, "playwright.sync_api": fake_playwright}): + spec = importlib.util.spec_from_file_location("e2e_browser_scenarios", SCENARIOS_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + with contextlib.redirect_stdout(io.StringIO()): + module.record("A1", "a title", True) + module.record_blocked("A2", "another title", "a prerequisite") + self.assertEqual(["A1", "A2"], [item["id"] for item in module.results]) + self.assertEqual(["PASS", "BLOCKED"], [item["status"] for item in module.results]) + + +class BrowserIsolationTest(unittest.TestCase): + def test_browser_bootstrap_uses_its_own_state_and_never_the_run_state(self): + scenario = load_scenario() + payload = { + "steps": [ + {"id": step_id, "title": step_id, "status": "PASS", "detail": ""} + for step_id in browser.EXPECTED_SCENARIO_IDS + ], + "context": dict(PERSISTED_CONTEXT), + } + with tempfile.TemporaryDirectory() as directory: + run_path = Path(directory) + state_dir = run_path / browser.BROWSER_STATE_DIRNAME + state_dir.mkdir() + (state_dir / "state.json").write_text(json.dumps({ + "config_id": 7, "join_code": "JOIN-CODE-SECRET", + "cookies": {"researcher": {"auth_token": "session-token"}}, + })) + website = run_path / "code4me2-server" / "src/website" + (website / "node_modules").mkdir(parents=True) + seen = {} + + def fake_workflow(scenario_, *, only=None, run_dir=None, keep_stack=False, **kwargs): + seen["bootstrap_run_dir"] = Path(run_dir) + return 0 + + def fake_process_run(command, *, cwd, log_path, env=None, timeout=1200): + if len(command) > 1 and command[1] == "-u": + seen["scenarios_env"] = dict(env or {}) + Path(env["CODE4ME_E2E_BROWSER_RESULTS"]).write_text(json.dumps(payload)) + return 0 + + class _ReadyResponse: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *exception): + return False + + class _Child: + pid = 4242 + + def poll(self): + return None + + with patch.object(browser, "_require_browser_prerequisites", + return_value=("python", "node", "npm")), \ + patch.object(browser.workflow, "run_workflow", side_effect=fake_workflow), \ + patch.object(browser.process, "run", side_effect=fake_process_run), \ + patch.object(browser.process, "stop"), \ + patch.object(browser, "require_workspace", return_value=run_path), \ + patch.object(browser.subprocess, "Popen", return_value=_Child()), \ + patch.object(browser.urllib.request, "urlopen", return_value=_ReadyResponse()), \ + patch.object(browser, "_api_get", side_effect=_fake_api(_persisted_responses())), \ + patch.object(browser, "login_cookies", return_value={"auth_token": "fresh"}): + rc = browser.run(scenario, run_path) + self.assertEqual(0, rc) + self.assertEqual(state_dir.resolve(), seen["bootstrap_run_dir"].resolve()) + self.assertFalse((run_path / "state.json").exists()) + self.assertEqual( + list(browser.EXPECTED_SCENARIO_IDS), + json.loads(seen["scenarios_env"]["CODE4ME_E2E_BROWSER_EXPECTED_IDS"]), + ) + built = json.loads((run_path / "report.json").read_text()) + step = next(item for item in built["steps"] if item["id"] == "browser_test") + self.assertEqual("PASS", step["status"]) + self.assertTrue(all(step["details"]["persisted"].values())) + self.assertNotIn("JOIN-CODE-SECRET", (run_path / "browser-results.json").read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/nginx.prod.conf b/nginx.prod.conf new file mode 100644 index 0000000..34c7e46 --- /dev/null +++ b/nginx.prod.conf @@ -0,0 +1,163 @@ +# Production edge for the Code4Me research platform (production-readiness B-07/B-08). +# +# Baked into the website image (src/website/Dockerfile): it serves the built +# React bundle as static files and proxies /api/ to the backend service. TLS is +# terminated in front of this container (a cloud load balancer, Caddy, Traefik or +# the host's nginx); see docs/research-platform/PRODUCTION_DEPLOYMENT.md, which +# also shows how to terminate TLS here instead. + +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + sendfile on; + server_tokens off; + keepalive_timeout 65; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for" rt=$request_time'; + access_log /var/log/nginx/access.log main; + error_log /var/log/nginx/error.log warn; + + gzip on; + gzip_types text/plain text/css application/javascript application/json image/svg+xml; + + # Re-resolve the backend through Docker's DNS: a recreated backend container + # gets a new address, and a name resolved once at startup then answers 502 + # until this container restarts. `resolve` needs nginx 1.27.3+ (Dockerfile). + resolver 127.0.0.11 valid=10s ipv6=off; + + upstream backend { + zone backend 64k; + server backend:8008 resolve; + keepalive 32; + } + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + # Behind an external TLS edge $remote_addr is the edge, not the client. + # Uncomment and set the edge's address/CIDR so nginx recovers the real + # client from the edge's X-Forwarded-For before forwarding a single hop: + # set_real_ip_from 10.0.0.0/8; + # real_ip_header X-Forwarded-For; + # real_ip_recursive on; + + server { + listen 80; + server_name _; + + # Telemetry batches are up to 1 MB of events plus envelope; 16 MB leaves + # headroom without letting a client hold a huge body open. + client_max_body_size 16m; + + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options DENY always; + add_header Referrer-Policy strict-origin-when-cross-origin always; + + # Edge liveness for external monitors (no backend dependency). + # add_header is not inherited into a location that sets its own + # headers, so the security headers are repeated where needed. + location = /nginx-health { + access_log off; + add_header Content-Type text/plain; + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options DENY always; + add_header Referrer-Policy strict-origin-when-cross-origin always; + return 200 'ok'; + } + + # --- API ----------------------------------------------------------- + location /api/ { + proxy_pass http://backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + # Exactly ONE hop: the address this proxy saw. The backend trusts + # the header from this container (FORWARDED_ALLOW_IPS=*) and keys + # rate limits on it; forwarding the client-supplied chain + # ($proxy_add_x_forwarded_for) would let a client choose its key. + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 120s; + proxy_send_timeout 120s; + } + + # Release archive imports (admin only) are large, multi-minute uploads. + location /api/research/agents/releases/import { + client_max_body_size 300m; + proxy_request_buffering off; + proxy_pass http://backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900s; + proxy_send_timeout 900s; + } + + # Streamed model responses (research inference gateway, ACP relay): + # never buffer, and allow the backend's 600 s stream budget. + location ~ ^/api/(research/inference|acp|agent/inference)(/|$) { + proxy_pass http://backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 900s; + proxy_send_timeout 900s; + } + + # WebSocket completion/chat endpoints. + location /api/ws/ { + proxy_pass http://backend; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + } + + # --- Website (static React build) ----------------------------------- + root /usr/share/nginx/html; + + location /static/ { + expires 1y; + add_header Cache-Control "public, immutable"; + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options DENY always; + add_header Referrer-Policy strict-origin-when-cross-origin always; + try_files $uri =404; + } + + location = /index.html { + add_header Cache-Control "no-cache"; + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options DENY always; + add_header Referrer-Policy strict-origin-when-cross-origin always; + } + + location / { + try_files $uri /index.html; + } + } +} diff --git a/nginx.prod.tls.conf.example b/nginx.prod.tls.conf.example new file mode 100644 index 0000000..ea312da --- /dev/null +++ b/nginx.prod.tls.conf.example @@ -0,0 +1,27 @@ +# Optional: terminate TLS in the website/nginx container itself. +# +# 1. Obtain a certificate (e.g. certbot on the host) and mount it: +# volumes: +# - /etc/letsencrypt/live//fullchain.pem:/etc/nginx/certs/fullchain.pem:ro +# - /etc/letsencrypt/live//privkey.pem:/etc/nginx/certs/privkey.pem:ro +# - ./nginx.prod.tls.conf:/etc/nginx/nginx.conf:ro +# ports: ["80:80", "443:443"] +# 2. Copy nginx.prod.conf to nginx.prod.tls.conf, change `listen 80;` to the +# two server blocks below, and keep every location block unchanged. +# +# server { +# listen 80; +# server_name ; +# return 301 https://$host$request_uri; +# } +# +# server { +# listen 443 ssl http2; +# server_name ; +# ssl_certificate /etc/nginx/certs/fullchain.pem; +# ssl_certificate_key /etc/nginx/certs/privkey.pem; +# ssl_protocols TLSv1.2 TLSv1.3; +# ssl_prefer_server_ciphers on; +# add_header Strict-Transport-Security "max-age=31536000" always; +# ... (the location blocks from nginx.prod.conf) ... +# } diff --git a/openapi.json b/openapi.json index dd3566d..aea274e 100644 --- a/openapi.json +++ b/openapi.json @@ -8,16 +8,22 @@ "paths": { "/api/session/acquire": { "get": { - "tags": ["Session"], + "tags": [ + "Session" + ], "summary": "Acquire Session", - "description": "Acquire or create a session token using the provided auth token.\n\n- If the auth token is missing or invalid, return 401.\n- If no session is associated yet, create one and store it in Redis.", + "description": "Acquire or create a session token linked to the provided auth token.\n\nArgs:\n app (App): Injected FastAPI app instance.\n auth_token (str): Auth token from the client's cookies.\n\nReturns:\n JsonResponseWithStatus: Contains the session token or appropriate error response.\n\nBehavior:\n - Returns 401 if auth token is missing or invalid.\n - If no session token exists for the auth token, creates a new session.\n - Sets the session token as an HttpOnly cookie with expiration.", "operationId": "acquire_session_api_session_acquire_get", "parameters": [ { "name": "auth_token", "in": "cookie", "required": false, - "schema": { "type": "string", "default": "", "title": "Auth Token" } + "schema": { + "type": "string", + "default": "", + "title": "Auth Token" + } } ], "responses": { @@ -41,10 +47,22 @@ }, "description": "Unauthorized" }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UserNotFoundError" + } + } + }, + "description": "Not Found" + }, "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -52,7 +70,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -60,7 +80,9 @@ "500": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/AcquireSessionError" } + "schema": { + "$ref": "#/components/schemas/AcquireSessionError" + } } }, "description": "Internal Server Error" @@ -70,23 +92,31 @@ }, "/api/project/create": { "post": { - "tags": ["Project"], + "tags": [ + "Project" + ], "summary": "Create Project", - "description": "Create a new project\n1. Validate the provided session token\n2. If valid, create a project and return the project token\n3. If invalid, return an appropriate error response", + "description": "Create a new project for an authenticated user.\n\nSteps:\n1. Validate the provided auth token to get session info.\n2. Verify the session token is valid.\n3. Create a new project in the database.\n4. Associate the project with the session and the user.\n5. Store project metadata in Redis.\n6. Return the project token as an HttpOnly cookie in the response.\n\nArgs:\n project_to_create: Project data provided in the request body.\n app: FastAPI dependency to access the app context.\n auth_token: Auth token passed as a cookie for authentication.\n\nReturns:\n JsonResponseWithStatus: JSON response with project token or error details.", "operationId": "create_project_api_project_create_post", "parameters": [ { "name": "auth_token", "in": "cookie", "required": false, - "schema": { "type": "string", "default": "", "title": "Auth Token" } + "schema": { + "type": "string", + "default": "", + "title": "Auth Token" + } } ], "requestBody": { "required": true, "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/CreateProject" } + "schema": { + "$ref": "#/components/schemas/CreateProject" + } } } }, @@ -129,10 +159,30 @@ }, "description": "Unauthorized" }, + "404": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/UserNotFoundError" + }, + { + "$ref": "#/components/schemas/SessionNotFoundError" + } + ], + "title": "Response 404 Create Project Api Project Create Post" + } + } + }, + "description": "Not Found" + }, "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -140,7 +190,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -148,7 +200,9 @@ "500": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/CreateProjectError" } + "schema": { + "$ref": "#/components/schemas/CreateProjectError" + } } }, "description": "Internal Server Error" @@ -158,23 +212,31 @@ }, "/api/project/activate": { "put": { - "tags": ["Project"], + "tags": [ + "Project" + ], "summary": "Activate Project", - "description": "Activates the project by following these steps:\n1. Validate the provided auth token\n2. If valid, return confirmation\n3. If invalid, return an appropriate error response\n4. The project might exist in redis or in the database, if it is in the database, it should be fetched from there and put in redis\nif it is in the redis, its expiration time should be updated.", + "description": "Activate a project for a user session by performing the following:\n\n1. Validate the auth token and session token from cookies.\n2. If the project exists in Redis, update its expiration time.\n3. If the project is not found in Redis, fetch it from the database and cache it.\n4. Associate the project with the user session and user in the database if not already linked.\n5. Return a success response with the project token set as an HttpOnly cookie.\n\nArgs:\n activate_project_request: Request data containing the project ID to activate.\n app: FastAPI dependency injection to get app context.\n auth_token: Auth token from cookie to validate user identity.\n\nReturns:\n JsonResponseWithStatus: Success or error response with appropriate status and messages.", "operationId": "activate_project_api_project_activate_put", "parameters": [ { "name": "auth_token", "in": "cookie", "required": false, - "schema": { "type": "string", "default": "", "title": "Auth Token" } + "schema": { + "type": "string", + "default": "", + "title": "Auth Token" + } } ], "requestBody": { "required": true, "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ActivateProject" } + "schema": { + "$ref": "#/components/schemas/ActivateProject" + } } } }, @@ -197,7 +259,9 @@ { "$ref": "#/components/schemas/InvalidOrExpiredSessionToken" }, - { "$ref": "#/components/schemas/InvalidOrExpiredAuthToken" } + { + "$ref": "#/components/schemas/InvalidOrExpiredAuthToken" + } ], "title": "Response 401 Activate Project Api Project Activate Put" } @@ -205,10 +269,33 @@ }, "description": "Unauthorized" }, + "404": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/UserNotFoundError" + }, + { + "$ref": "#/components/schemas/SessionNotFoundError" + }, + { + "$ref": "#/components/schemas/ProjectNotFoundError" + } + ], + "title": "Response 404 Activate Project Api Project Activate Put" + } + } + }, + "description": "Not Found" + }, "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -216,7 +303,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -236,19 +325,17 @@ }, "/api/user/create": { "post": { - "tags": ["User"], + "tags": [ + "User" + ], "summary": "Create User", - "description": "Create a new user in the system.\n\nArgs:\n user_to_create (Union[Queries.CreateUser, Queries.CreateUserOauth]):\n The user data to create, can be standard or OAuth-based.\n app (App):\n The application instance, injected by FastAPI's dependency system.\n\nReturns:\n JsonResponseWithStatus: Response with status code and content.\n\nSteps:\n 1. Check if the user already exists by email.\n 2. If OAuth, verify the JWT token and email.\n 3. Create the user in the database if not exists.\n 4. Send verification email.\n 5. Return appropriate response.", + "description": "Create a new user with email and password.\n\nArgs:\n user_to_create (CreateUser):\n User data from the request body.\n app (App):\n Application context with access to database and services.\n\nReturns:\n JsonResponseWithStatus: JSON response indicating success or failure.\n\nFlow:\n 1. Check if a user already exists with the given email.\n 2. Insert the new user into the database.\n 3. Send a verification email via Celery.\n 4. Return HTTP 201 with the new user ID.", "operationId": "create_user_api_user_create_post", "requestBody": { "content": { "application/json": { "schema": { - "anyOf": [ - { "$ref": "#/components/schemas/CreateUser" }, - { "$ref": "#/components/schemas/CreateUserOauth" } - ], - "title": "User To Create" + "$ref": "#/components/schemas/CreateUser" } } }, @@ -275,16 +362,6 @@ } } }, - "401": { - "description": "Unauthorized", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InvalidOrExpiredJWTToken" - } - } - } - }, "409": { "description": "Conflict", "content": { @@ -299,7 +376,9 @@ "description": "Unprocessable Entity", "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } } }, @@ -307,7 +386,9 @@ "description": "Too Many Requests", "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } } }, @@ -315,7 +396,9 @@ "description": "Internal Server Error", "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/CreateUserError" } + "schema": { + "$ref": "#/components/schemas/CreateUserError" + } } } } @@ -324,23 +407,31 @@ }, "/api/user/update": { "put": { - "tags": ["User"], + "tags": [ + "User" + ], "summary": "Update User", - "description": "Update the currently authenticated user's data.\n\nArgs:\n- user_to_update: Pydantic model containing fields to update.\n- app: Application context, injected by FastAPI.\n- auth_token: Authentication token stored in browser cookies.\n\nReturns:\n- JSON response with updated user information if successful.\n- Appropriate error response if auth token is missing or invalid.", + "description": "Update the currently authenticated user's data.\n\nArgs:\n user_to_update (Queries.UpdateUser): Fields the user wants to update.\n app (App): Injected application instance with DB and Redis access.\n auth_token (str): Auth token stored in the user's browser cookies.\n\nReturns:\n JsonResponseWithStatus: Contains updated user data or error info.", "operationId": "update_user_api_user_update_put", "parameters": [ { "name": "auth_token", "in": "cookie", "required": false, - "schema": { "type": "string", "default": "", "title": "Auth Token" } + "schema": { + "type": "string", + "default": "", + "title": "Auth Token" + } } ], "requestBody": { "required": true, "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/UpdateUser" } + "schema": { + "$ref": "#/components/schemas/UpdateUser" + } } } }, @@ -375,6 +466,16 @@ }, "description": "Unauthorized" }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UserNotFoundError" + } + } + }, + "description": "Not Found" + }, "409": { "content": { "application/json": { @@ -388,7 +489,9 @@ "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -396,7 +499,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -404,7 +509,9 @@ "500": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/UpdateUserError" } + "schema": { + "$ref": "#/components/schemas/UpdateUserError" + } } }, "description": "Internal Server Error" @@ -414,9 +521,11 @@ }, "/api/user/delete": { "delete": { - "tags": ["User"], + "tags": [ + "User" + ], "summary": "Delete User", - "description": "Delete the authenticated user's account and optionally their data.\n\nArgs:\n delete_data (bool): Flag indicating whether to delete associated data (default: False).\n auth_token (str): Authentication token stored in browser cookies.\n app (App): Application instance with access to database and session managers.\n\nReturns:\n JsonResponseWithStatus: A success message or an appropriate error response.", + "description": "Delete the authenticated user's account together with all data collected about it.\n\nArgs:\n delete_data (bool): Ignored; kept so existing clients keep working.\n auth_token (str): Auth token provided in cookies to authenticate the user.\n app (App): Dependency-injected app instance with DB and Redis access.\n\nReturns:\n JsonResponseWithStatus: A success or error response depending on the outcome.", "operationId": "delete_user_api_user_delete_delete", "parameters": [ { @@ -425,17 +534,21 @@ "required": false, "schema": { "type": "boolean", - "description": "Delete user's data", + "description": "Deprecated and ignored: deleting an account always erases its data.", "default": false, "title": "Delete Data" }, - "description": "Delete user's data" + "description": "Deprecated and ignored: deleting an account always erases its data." }, { "name": "auth_token", "in": "cookie", "required": false, - "schema": { "type": "string", "default": "", "title": "Auth Token" } + "schema": { + "type": "string", + "default": "", + "title": "Auth Token" + } } ], "responses": { @@ -462,15 +575,29 @@ "404": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/UserNotFoundError" } + "schema": { + "$ref": "#/components/schemas/UserNotFoundError" + } } }, "description": "Not Found" }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteUserBlockedError" + } + } + }, + "description": "Conflict" + }, "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -478,7 +605,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -486,7 +615,9 @@ "500": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/DeleteUserError" } + "schema": { + "$ref": "#/components/schemas/DeleteUserError" + } } }, "description": "Internal Server Error" @@ -496,21 +627,18 @@ }, "/api/user/authenticate": { "post": { - "tags": ["User", "Authentication"], + "tags": [ + "User", + "Authentication" + ], "summary": "Authenticate User", - "description": "Authenticate a user via either OAuth (JWT token) or traditional email/password.\n\nThis endpoint supports two methods of authentication:\n1. OAuth Authentication:\n - The input contains a JWT token from an OAuth provider (Google).\n - The token's validity is verified.\n - If valid, the user is fetched by email from the database.\n - A session auth token is created and returned as a cookie.\n2. Email/Password Authentication:\n - The input contains user email and password.\n - Credentials are verified against the database.\n - If valid, a session auth token is created and returned as a cookie.\n\nArgs:\n user_to_authenticate: Union of OAuth token or email/password credentials.\n app: FastAPI dependency to access the application context.\n\nReturns:\n JsonResponseWithStatus: A JSON response containing the authenticated user info\n and a session auth token cookie on success, or an error response otherwise.", + "description": "Authenticate a user using email/password.\n\nThis endpoint supports:\n- Email/Password: Verifies credentials against the database.\n\nArgs:\n user_to_authenticate (AuthenticateUserEmailPassword):\n An email/password object.\n app (App): FastAPI dependency that provides access to DB and config.\n\nReturns:\n JsonResponseWithStatus: Authenticated user info + auth token cookie,\n or an error response.", "operationId": "authenticate_user_api_user_authenticate_post", "requestBody": { "content": { "application/json": { "schema": { - "anyOf": [ - { - "$ref": "#/components/schemas/AuthenticateUserEmailPassword" - }, - { "$ref": "#/components/schemas/AuthenticateUserOAuth" } - ], - "title": "User To Authenticate" + "$ref": "#/components/schemas/AuthenticateUserEmailPassword" } } }, @@ -532,11 +660,7 @@ "content": { "application/json": { "schema": { - "anyOf": [ - { "$ref": "#/components/schemas/InvalidOrExpiredJWTToken" }, - { "$ref": "#/components/schemas/InvalidEmailOrPassword" } - ], - "title": "Response 401 Authenticate User Api User Authenticate Post" + "$ref": "#/components/schemas/InvalidEmailOrPassword" } } } @@ -545,7 +669,9 @@ "description": "Not Found", "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ConfigNotFound" } + "schema": { + "$ref": "#/components/schemas/ConfigNotFound" + } } } }, @@ -553,7 +679,9 @@ "description": "Unprocessable Entity", "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } } }, @@ -561,7 +689,9 @@ "description": "Too Many Requests", "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } } }, @@ -580,16 +710,23 @@ }, "/api/user/verify/check": { "get": { - "tags": ["User", "User Verification"], + "tags": [ + "User", + "User Verification" + ], "summary": "Check Verification", - "description": "Check if the user is verified", + "description": "Check if the currently authenticated user has verified their email.\n\nParameters:\n- app (App): Dependency-injected application context providing access to services.\n- auth_token (str): Authentication token retrieved from the user's cookie.\n\nReturns:\n- JsonResponseWithStatus: User verification status or appropriate error message.", "operationId": "check_verification_api_user_verify_check_get", "parameters": [ { "name": "auth_token", "in": "cookie", "required": false, - "schema": { "type": "string", "default": "", "title": "Auth Token" } + "schema": { + "type": "string", + "default": "", + "title": "Auth Token" + } } ], "responses": { @@ -616,7 +753,9 @@ "404": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/UserNotFoundError" } + "schema": { + "$ref": "#/components/schemas/UserNotFoundError" + } } }, "description": "Not Found" @@ -624,7 +763,9 @@ "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -632,7 +773,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -652,16 +795,23 @@ }, "/api/user/verify/resend": { "post": { - "tags": ["User", "User Verification"], + "tags": [ + "User", + "User Verification" + ], "summary": "Resend Verification Email", - "description": "Resend verification email to the user", + "description": "Resend a verification email to the user if not already verified.\n\nParameters:\n- app (App): Application context for accessing services.\n- auth_token (str): Auth token from cookie identifying the user.\n\nReturns:\n- JsonResponseWithStatus: Success confirmation or error message.", "operationId": "resend_verification_email_api_user_verify_resend_post", "parameters": [ { "name": "auth_token", "in": "cookie", "required": false, - "schema": { "type": "string", "default": "", "title": "Auth Token" } + "schema": { + "type": "string", + "default": "", + "title": "Auth Token" + } } ], "responses": { @@ -688,7 +838,9 @@ "404": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/UserNotFoundError" } + "schema": { + "$ref": "#/components/schemas/UserNotFoundError" + } } }, "description": "Not Found" @@ -696,7 +848,9 @@ "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -704,7 +858,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -718,16 +874,29 @@ } }, "description": "Internal Server Error" + }, + "502": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResendVerificationEmailError" + } + } + }, + "description": "Bad Gateway" } } } }, "/api/user/verify/": { - "post": { - "tags": ["User", "User Verification"], + "get": { + "tags": [ + "User", + "User Verification" + ], "summary": "Verify Email", - "description": "Verify user email with the provided token", - "operationId": "verify_email_api_user_verify__post", + "description": "Verify the user's email address using the provided verification token.\n\nParameters:\n- token (str): Token for email verification, passed via query parameter.\n- app (App): Application context for accessing services.\n\nReturns:\n- HTMLResponseWithStatus: HTML response indicating verification outcome.", + "operationId": "verify_email_api_user_verify__get", "parameters": [ { "name": "token", @@ -747,7 +916,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VerifyUserPostHTMLResponse" + "$ref": "#/components/schemas/VerifyUserGetHTMLResponse" } } } @@ -765,7 +934,9 @@ "404": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/UserNotFoundError" } + "schema": { + "$ref": "#/components/schemas/UserNotFoundError" + } } }, "description": "Not Found" @@ -773,7 +944,9 @@ "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -781,7 +954,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -789,7 +964,9 @@ "500": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/VerifyUserError" } + "schema": { + "$ref": "#/components/schemas/VerifyUserError" + } } }, "description": "Internal Server Error" @@ -799,9 +976,11 @@ }, "/api/completion/request": { "post": { - "tags": ["Completion"], + "tags": [ + "Completion" + ], "summary": "Request Completion", - "description": "Request code completions based on provided context.", + "description": "Handle a code completion request.\n\nSteps:\n- Authenticate session, user, and project tokens via Redis.\n- Redact any secrets from the context before processing.\n- Prepare optional Celery tasks for storing context and telemetry.\n- Aggregate multi-file context if available.\n- Run completion models concurrently using a thread pool.\n- Queue database update tasks asynchronously using Celery.\n- Return completion results or appropriate error responses.", "operationId": "request_completion_api_completion_request_post", "parameters": [ { @@ -829,7 +1008,9 @@ "required": true, "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/RequestCompletion" } + "schema": { + "$ref": "#/components/schemas/RequestCompletion" + } } } }, @@ -849,9 +1030,6 @@ "application/json": { "schema": { "anyOf": [ - { - "$ref": "#/components/schemas/InvalidOrExpiredAuthToken" - }, { "$ref": "#/components/schemas/InvalidOrExpiredSessionToken" }, @@ -868,7 +1046,9 @@ "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -876,7 +1056,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -896,9 +1078,11 @@ }, "/api/completion/feedback": { "post": { - "tags": ["Completion"], + "tags": [ + "Completion" + ], "summary": "Submit Completion Feedback", - "description": "Submit feedback on a generated completion.", + "description": "Submit feedback on a generated completion.\n\nThis endpoint validates the user's session and project tokens against Redis,\nverifies the user's authorization to provide feedback on the given query,\nand enqueues asynchronous tasks to update generation status and optionally\nsave ground truth feedback.\n\nParameters:\n- feedback: The feedback data submitted by the user, including acceptance\n status and optional ground truth.\n- app: Dependency-injected application instance providing DB and Redis access.\n- session_token: Session cookie used to authenticate the user session.\n- project_token: Project cookie used to authorize project-level access.\n\nReturns:\n- JsonResponseWithStatus: A response indicating success or describing errors,\n including authorization failures, missing records, or internal errors.", "operationId": "submit_completion_feedback_api_completion_feedback_post", "parameters": [ { @@ -926,7 +1110,9 @@ "required": true, "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/FeedbackCompletion" } + "schema": { + "$ref": "#/components/schemas/FeedbackCompletion" + } } } }, @@ -949,9 +1135,6 @@ { "$ref": "#/components/schemas/InvalidOrExpiredSessionToken" }, - { - "$ref": "#/components/schemas/InvalidOrExpiredAuthToken" - }, { "$ref": "#/components/schemas/InvalidOrExpiredProjectToken" } @@ -985,7 +1168,9 @@ "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -993,7 +1178,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -1013,9 +1200,11 @@ }, "/api/completion/{query_id}": { "get": { - "tags": ["Completion"], + "tags": [ + "Completion" + ], "summary": "Get Completions By Query", - "description": "Get completions for a specific query ID.", + "description": "Retrieve code completions associated with a specific query ID.\n\nThis endpoint validates the user's session token and authorization,\nensures the requested query exists and belongs to the requesting user,\nand then fetches all the associated completion generations from the database.\n\nParameters:\n- query_id (UUID): The unique identifier for the meta query whose completions are requested.\n- app (App, dependency): The application instance providing database and Redis access.\n- session_token (str, cookie): The session token cookie for user authentication.\n\nReturns:\n- JsonResponseWithStatus: JSON response with HTTP status and either the completions data\n or an error response detailing the failure reason.\n\nPossible responses:\n- 200: Successfully retrieved completions for the given query ID.\n- 401: Session token is invalid, expired, or missing.\n- 403: User does not have access rights to the requested query.\n- 404: The requested query ID does not exist.\n- 422, 429: Various client errors (validation or rate limiting).\n- 500: Internal server error while retrieving completions.", "operationId": "get_completions_by_query_api_completion__query_id__get", "parameters": [ { @@ -1073,7 +1262,9 @@ "404": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/QueryNotFoundError" } + "schema": { + "$ref": "#/components/schemas/QueryNotFoundError" + } } }, "description": "Not Found" @@ -1081,7 +1272,9 @@ "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -1089,7 +1282,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -1109,9 +1304,11 @@ }, "/api/completion/multi-file-context/update": { "post": { - "tags": ["Multi File Context"], + "tags": [ + "Multi File Context" + ], "summary": "Update Multi File Context", - "description": "Update the context for a specific query ID.", + "description": "Endpoint to update multi-file context content for a given project and session.\n\nSteps:\n- Validate session token and retrieve session info from Redis.\n- Validate project token and retrieve project info from Redis.\n- Apply line-level context updates to the existing multi-file contexts.\n- Redact secrets in the updated content lines.\n- Remove any files with empty content.\n- Update the context change logs accordingly.\n- Store the updated project info back in Redis.\n- Return the updated context in the response.\n\nArgs:\n context_update: UpdateMultiFileContext model containing the changes to apply.\n app: The application instance (dependency-injected).\n session_token: Session token from cookies.\n project_token: Project token from cookies.\n\nReturns:\n JsonResponseWithStatus containing the updated multi-file context or an error response.", "operationId": "update_multi_file_context_api_completion_multi_file_context_update_post", "parameters": [ { @@ -1177,7 +1374,9 @@ "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -1185,7 +1384,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -1205,7 +1406,9 @@ }, "/api/chat/request": { "post": { - "tags": ["Chat"], + "tags": [ + "Chat" + ], "summary": "Request Chat Completion", "description": "Request chat completions based on provided messages.\n\nThe contract here is that the request always contains all the history of the chat as well\n\nWe do this because it could be that the user modifies the chat history in the frontend\nand we want to ensure that the chat completions are based on the latest state of the chat.\n\nTake the case where the user edits a message in the chat history midway through a chat.\nWe don't want to be generating a completion based on the old state of the chat.", "operationId": "request_chat_completion_api_chat_request_post", @@ -1235,7 +1438,9 @@ "required": true, "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/RequestChatCompletion" } + "schema": { + "$ref": "#/components/schemas/RequestChatCompletion" + } } } }, @@ -1244,7 +1449,9 @@ "description": "Successful Response", "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ChatHistoryResponse" } + "schema": { + "$ref": "#/components/schemas/ChatHistoryResponse" + } } } }, @@ -1253,9 +1460,6 @@ "application/json": { "schema": { "anyOf": [ - { - "$ref": "#/components/schemas/InvalidOrExpiredAuthToken" - }, { "$ref": "#/components/schemas/InvalidOrExpiredSessionToken" }, @@ -1272,7 +1476,9 @@ "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -1280,7 +1486,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -1300,16 +1508,21 @@ }, "/api/chat/get/{page_number}": { "get": { - "tags": ["Chat"], + "tags": [ + "Chat" + ], "summary": "Get Chat History", - "description": "Get the complete chat history for", + "description": "Retrieve a page of chat history for a specific project associated with the current session.\n\nParameters:\n- app (App): Dependency-injected FastAPI application instance.\n- page_number (int): The page number of the chat history to retrieve.\n- session_token (str): Session token stored in a cookie.\n- project_token (str): Project token stored in a cookie.\n\nReturns:\n- JsonResponseWithStatus: Paginated chat history or an error response.", "operationId": "get_chat_history_api_chat_get__page_number__get", "parameters": [ { "name": "page_number", "in": "path", "required": true, - "schema": { "type": "integer", "title": "Page Number" } + "schema": { + "type": "integer", + "title": "Page Number" + } }, { "name": "session_token", @@ -1348,9 +1561,6 @@ "application/json": { "schema": { "anyOf": [ - { - "$ref": "#/components/schemas/InvalidOrExpiredAuthToken" - }, { "$ref": "#/components/schemas/InvalidOrExpiredSessionToken" }, @@ -1377,7 +1587,9 @@ "404": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/QueryNotFoundError" } + "schema": { + "$ref": "#/components/schemas/QueryNotFoundError" + } } }, "description": "Not Found" @@ -1385,7 +1597,9 @@ "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -1393,7 +1607,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -1413,7 +1629,9 @@ }, "/api/chat/delete/{chat_id}": { "delete": { - "tags": ["Chat"], + "tags": [ + "Chat" + ], "summary": "Delete Chat", "description": "Delete a specific chat by its ID.\nValidates that the user has access to the chat through their session and project tokens.", "operationId": "delete_chat_api_chat_delete__chat_id__delete", @@ -1422,7 +1640,11 @@ "name": "chat_id", "in": "path", "required": true, - "schema": { "type": "string", "format": "uuid", "title": "Chat Id" } + "schema": { + "type": "string", + "format": "uuid", + "title": "Chat Id" + } }, { "name": "session_token", @@ -1461,9 +1683,6 @@ "application/json": { "schema": { "anyOf": [ - { - "$ref": "#/components/schemas/InvalidOrExpiredAuthToken" - }, { "$ref": "#/components/schemas/InvalidOrExpiredSessionToken" }, @@ -1490,7 +1709,9 @@ "404": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/QueryNotFoundError" } + "schema": { + "$ref": "#/components/schemas/QueryNotFoundError" + } } }, "description": "Not Found" @@ -1498,7 +1719,9 @@ "422": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Unprocessable Entity" @@ -1506,7 +1729,9 @@ "429": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/ErrorResponse" } + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } } }, "description": "Too Many Requests" @@ -1514,7 +1739,9 @@ "500": { "content": { "application/json": { - "schema": { "$ref": "#/components/schemas/DeleteChatError" } + "schema": { + "$ref": "#/components/schemas/DeleteChatError" + } } }, "description": "Internal Server Error" @@ -1523,23 +1750,18 @@ } }, "/api/ping": { - "get": { - "summary": "Ping", - "operationId": "ping_api_ping_head", - "responses": { - "200": { - "description": "Successful Response", - "content": { "application/json": { "schema": {} } } - } - } - }, "head": { "summary": "Ping", + "description": "Lightweight health check endpoint.\n\nReturns:\n dict: A simple response indicating the service is available.", "operationId": "ping_api_ping_head", "responses": { "200": { "description": "Successful Response", - "content": { "application/json": { "schema": {} } } + "content": { + "application/json": { + "schema": {} + } + } } } } @@ -1572,7 +1794,9 @@ } }, "type": "object", - "required": ["session_token"], + "required": [ + "session_token" + ], "title": "AcquireSessionGetResponse" }, "ActivateProject": { @@ -1585,7 +1809,9 @@ } }, "type": "object", - "required": ["project_id"], + "required": [ + "project_id" + ], "title": "ActivateProject" }, "ActivateProjectError": { @@ -1627,7 +1853,10 @@ } }, "type": "object", - "required": ["email", "password"], + "required": [ + "email", + "password" + ], "title": "AuthenticateUserEmailPassword" }, "AuthenticateUserError": { @@ -1641,22 +1870,6 @@ "type": "object", "title": "AuthenticateUserError" }, - "AuthenticateUserOAuth": { - "properties": { - "provider": { - "$ref": "#/components/schemas/Provider", - "description": "OAuth provider" - }, - "token": { - "type": "string", - "title": "Token", - "description": "OAuth token in JWT format" - } - }, - "type": "object", - "required": ["provider", "token"], - "title": "AuthenticateUserOAuth" - }, "AuthenticateUserPostResponse": { "properties": { "message": { @@ -1669,41 +1882,59 @@ "description": "User details" }, "config": { - "additionalProperties": true, - "type": "object", + "type": "string", "title": "Config", - "description": "Users config json string" + "description": "User's config HOCON string" } }, "type": "object", - "required": ["message", "user", "config"], + "required": [ + "message", + "user", + "config" + ], "title": "AuthenticateUserPostResponse" }, "BehavioralTelemetryData": { "properties": { "time_since_last_shown": { "anyOf": [ - { "type": "integer", "minimum": 0.0 }, - { "type": "null" } + { + "type": "integer", + "minimum": 0.0 + }, + { + "type": "null" + } ], "title": "Time Since Last Shown", "description": "Time since last completion shown (ms)" }, "time_since_last_accepted": { "anyOf": [ - { "type": "integer", "minimum": 0.0 }, - { "type": "null" } + { + "type": "integer", + "minimum": 0.0 + }, + { + "type": "null" + } ], "title": "Time Since Last Accepted", "description": "Time since last completion accepted (ms)" }, "typing_speed": { "anyOf": [ - { "type": "integer", "minimum": 0.0 }, - { "type": "null" } + { + "type": "number", + "minimum": 0.0 + }, + { + "type": "null" + } ], "title": "Typing Speed", - "description": "Typing speed (chars per minute)" + "description": "Typing speed" } }, "type": "object", @@ -1719,12 +1950,15 @@ "model_name": { "type": "string", "title": "Model Name", - "description": "Model name" + "description": "Model name that failed" } }, "type": "object", - "required": ["model_name"], - "title": "ChatCompletionErrorItem" + "required": [ + "model_name" + ], + "title": "ChatCompletionErrorItem", + "description": "Represents a failed attempt to generate a chat completion." }, "ChatCompletionItem": { "properties": { @@ -1748,7 +1982,7 @@ "type": "integer", "minimum": 0.0, "title": "Generation Time", - "description": "Generation time" + "description": "Generation time in ms" }, "confidence": { "type": "number", @@ -1758,7 +1992,7 @@ "was_accepted": { "type": "boolean", "title": "Was Accepted", - "description": "Whether the completion was accepted" + "description": "User acceptance status" } }, "type": "object", @@ -1771,30 +2005,37 @@ "was_accepted" ], "title": "ChatCompletionItem", - "description": "Represents a single model completion in a chat conversation" + "description": "Model-generated response in a chat conversation." }, "ChatHistoryItem": { "properties": { "user_message": { "$ref": "#/components/schemas/ChatMessageItem", - "description": "User's message" + "description": "User's input message" }, "assistant_responses": { "items": { "anyOf": [ - { "$ref": "#/components/schemas/ChatCompletionItem" }, - { "$ref": "#/components/schemas/ChatCompletionErrorItem" } + { + "$ref": "#/components/schemas/ChatCompletionItem" + }, + { + "$ref": "#/components/schemas/ChatCompletionErrorItem" + } ] }, "type": "array", "title": "Assistant Responses", - "description": "Assistant's responses from different models" + "description": "Assistant's model responses" } }, "type": "object", - "required": ["user_message", "assistant_responses"], + "required": [ + "user_message", + "assistant_responses" + ], "title": "ChatHistoryItem", - "description": "Represents a single turn in the chat conversation" + "description": "Represents a single user-assistant interaction in a chat session." }, "ChatHistoryResponse": { "properties": { @@ -1802,84 +2043,110 @@ "type": "string", "format": "uuid", "title": "Chat Id", - "description": "Chat ID" + "description": "Unique chat session ID" }, "title": { "type": "string", "title": "Title", - "description": "Chat title" + "description": "Title of the chat" }, "history": { - "items": { "$ref": "#/components/schemas/ChatHistoryItem" }, + "items": { + "$ref": "#/components/schemas/ChatHistoryItem" + }, "type": "array", "title": "History", - "description": "Chat conversation history" + "description": "List of message exchanges" } }, "type": "object", - "required": ["chat_id", "title", "history"], + "required": [ + "chat_id", + "title", + "history" + ], "title": "ChatHistoryResponse", - "description": "Response containing the entire chat history" + "description": "Complete chat conversation history for a session." }, "ChatHistoryResponsePage": { "properties": { "page": { "type": "integer", "title": "Page", - "description": "Current page number" + "description": "Current page index" }, "per_page": { "type": "integer", "title": "Per Page", - "description": "Number of records per page" + "description": "Number of items per page" }, "items": { - "items": { "$ref": "#/components/schemas/ChatHistoryResponse" }, + "items": { + "$ref": "#/components/schemas/ChatHistoryResponse" + }, "type": "array", "title": "Items", - "description": "List of chat history responses" + "description": "Paginated chat sessions" } }, "type": "object", - "required": ["page", "per_page", "items"], + "required": [ + "page", + "per_page", + "items" + ], "title": "ChatHistoryResponsePage", - "description": "A glorified list of chat history responses with pagination" + "description": "Paginated response containing multiple chat histories." }, "ChatMessageItem": { "properties": { "role": { "$ref": "#/components/schemas/ChatMessageRole", - "description": "Role of the message sender" + "description": "Sender's role" }, "content": { "type": "string", "title": "Content", - "description": "Content of the message" + "description": "Message content" }, "timestamp": { "type": "string", "format": "date-time", "title": "Timestamp", - "description": "When the message was sent" + "description": "Time the message was sent" }, "meta_query_id": { "anyOf": [ - { "type": "string", "format": "uuid" }, - { "type": "null" } + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } ], "title": "Meta Query Id", - "description": "Associated query ID if any" + "description": "Associated meta query ID, if applicable" } }, "type": "object", - "required": ["role", "content", "timestamp"], + "required": [ + "role", + "content", + "timestamp" + ], "title": "ChatMessageItem", - "description": "Represents a single message in a chat conversation" + "description": "Represents a single message in a chat conversation." }, "ChatMessageRole": { "type": "string", - "enum": ["user", "assistant", "system"], - "title": "ChatMessageRole" + "enum": [ + "user", + "assistant", + "system" + ], + "title": "ChatMessageRole", + "description": "Enumeration of roles within a chat conversation." }, "CompletionErrorItem": { "properties": { @@ -1891,12 +2158,15 @@ "model_name": { "type": "string", "title": "Model Name", - "description": "Model name" + "description": "Model name that failed" } }, "type": "object", - "required": ["model_name"], - "title": "CompletionErrorItem" + "required": [ + "model_name" + ], + "title": "CompletionErrorItem", + "description": "Represents an error during code completion." }, "CompletionFeedbackPostResponse": { "properties": { @@ -1911,7 +2181,9 @@ } }, "type": "object", - "required": ["data"], + "required": [ + "data" + ], "title": "CompletionFeedbackPostResponse" }, "CompletionPostResponse": { @@ -1927,7 +2199,9 @@ } }, "type": "object", - "required": ["data"], + "required": [ + "data" + ], "title": "CompletionPostResponse" }, "ConfigNotFound": { @@ -1943,7 +2217,11 @@ }, "ContextChangeType": { "type": "string", - "enum": ["update", "insert", "remove"], + "enum": [ + "update", + "insert", + "remove" + ], "title": "ContextChangeType" }, "ContextData": { @@ -1959,21 +2237,42 @@ "description": "Code after cursor" }, "file_name": { - "anyOf": [{ "type": "string" }, { "type": "null" }], + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "File Name", "description": "File name", "default": "unknown_file" }, "selected_text": { - "anyOf": [{ "type": "string" }, { "type": "null" }], + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Selected Text", "description": "Selected text in editor", "default": "" }, "context_files": { "anyOf": [ - { "items": { "type": "string" }, "type": "array" }, - { "type": "null" } + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } ], "title": "Context Files", "description": "Context files to consider", @@ -1981,7 +2280,10 @@ } }, "type": "object", - "required": ["prefix", "suffix"], + "required": [ + "prefix", + "suffix" + ], "title": "ContextData" }, "ContextualTelemetryData": { @@ -2005,37 +2307,64 @@ "description": "Programming language ID" }, "file_path": { - "anyOf": [{ "type": "string" }, { "type": "null" }], + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "File Path", "description": "Path to the file being edited" }, "caret_line": { "anyOf": [ - { "type": "integer", "minimum": 0.0 }, - { "type": "null" } + { + "type": "integer", + "minimum": 0.0 + }, + { + "type": "null" + } ], "title": "Caret Line", "description": "Line number of cursor" }, "document_char_length": { "anyOf": [ - { "type": "integer", "minimum": 0.0 }, - { "type": "null" } + { + "type": "integer", + "minimum": 0.0 + }, + { + "type": "null" + } ], "title": "Document Char Length", "description": "Document length in characters" }, "relative_document_position": { "anyOf": [ - { "type": "number", "maximum": 1.0, "minimum": 0.0 }, - { "type": "null" } + { + "type": "number", + "maximum": 1.0, + "minimum": 0.0 + }, + { + "type": "null" + } ], "title": "Relative Document Position", "description": "Cursor position as fraction of document" } }, "type": "object", - "required": ["version_id", "trigger_type_id", "language_id"], + "required": [ + "version_id", + "trigger_type_id", + "language_id" + ], "title": "ContextualTelemetryData" }, "CreateProject": { @@ -2049,7 +2378,9 @@ } }, "type": "object", - "required": ["project_name"], + "required": [ + "project_name" + ], "title": "CreateProject" }, "CreateProjectError": { @@ -2077,7 +2408,9 @@ } }, "type": "object", - "required": ["project_token"], + "required": [ + "project_token" + ], "title": "CreateProjectPostResponse" }, "CreateUser": { @@ -2114,7 +2447,12 @@ } }, "type": "object", - "required": ["email", "name", "password", "config_id"], + "required": [ + "email", + "name", + "password", + "config_id" + ], "title": "CreateUser" }, "CreateUserError": { @@ -2128,59 +2466,6 @@ "type": "object", "title": "CreateUserError" }, - "CreateUserOauth": { - "properties": { - "email": { - "type": "string", - "maxLength": 50, - "minLength": 3, - "format": "email", - "title": "Email", - "description": "User's email address" - }, - "name": { - "type": "string", - "maxLength": 50, - "minLength": 3, - "title": "Name", - "description": "User's full name" - }, - "password": { - "type": "string", - "maxLength": 50, - "minLength": 8, - "format": "password", - "title": "Password", - "description": "User's password (will be hashed)", - "writeOnly": true - }, - "config_id": { - "type": "integer", - "minimum": 0.0, - "title": "Config Id", - "description": "Configuration ID to use" - }, - "token": { - "type": "string", - "title": "Token", - "description": "JWT token for authentication" - }, - "provider": { - "$ref": "#/components/schemas/Provider", - "description": "OAuth provider (Google, Microsoft, etc.)" - } - }, - "type": "object", - "required": [ - "email", - "name", - "password", - "config_id", - "token", - "provider" - ], - "title": "CreateUserOauth" - }, "CreateUserPostResponse": { "properties": { "message": { @@ -2196,7 +2481,9 @@ } }, "type": "object", - "required": ["user_id"], + "required": [ + "user_id" + ], "title": "CreateUserPostResponse" }, "DeleteChatError": { @@ -2209,7 +2496,7 @@ }, "type": "object", "title": "DeleteChatError", - "description": "Error response for chat deletion failure" + "description": "Error response for chat deletion failure." }, "DeleteChatSuccessResponse": { "properties": { @@ -2221,7 +2508,18 @@ }, "type": "object", "title": "DeleteChatSuccessResponse", - "description": "Success response for chat deletion" + "description": "Acknowledgement response for successful chat deletion." + }, + "DeleteUserBlockedError": { + "properties": { + "message": { + "type": "string", + "title": "Message", + "default": "This account owns research studies or agent profiles, so it cannot delete itself." + } + }, + "type": "object", + "title": "DeleteUserBlockedError" }, "DeleteUserDeleteResponse": { "properties": { @@ -2254,7 +2552,9 @@ } }, "type": "object", - "required": ["message"], + "required": [ + "message" + ], "title": "ErrorResponse" }, "FeedbackCompletion": { @@ -2277,13 +2577,24 @@ "description": "Whether completion was accepted" }, "ground_truth": { - "anyOf": [{ "type": "string" }, { "type": "null" }], + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Ground Truth", "description": "Actual code if available" } }, "type": "object", - "required": ["meta_query_id", "model_id", "was_accepted"], + "required": [ + "meta_query_id", + "model_id", + "was_accepted" + ], "title": "FeedbackCompletion" }, "FeedbackRecordingError": { @@ -2316,14 +2627,21 @@ "description": "End line number" }, "new_lines": { - "items": { "type": "string" }, + "items": { + "type": "string" + }, "type": "array", "title": "New Lines", "description": "New lines of code" } }, "type": "object", - "required": ["change_type", "start_line", "end_line", "new_lines"], + "required": [ + "change_type", + "start_line", + "end_line", + "new_lines" + ], "title": "FileContextChangeData" }, "GenerateChatCompletionsError": { @@ -2384,7 +2702,9 @@ } }, "type": "object", - "required": ["user_is_verified"], + "required": [ + "user_is_verified" + ], "title": "GetVerificationGetResponse" }, "InvalidEmailOrPassword": { @@ -2409,17 +2729,6 @@ "type": "object", "title": "InvalidOrExpiredAuthToken" }, - "InvalidOrExpiredJWTToken": { - "properties": { - "message": { - "type": "string", - "title": "Message", - "default": "Invalid or expired token!" - } - }, - "type": "object", - "title": "InvalidOrExpiredJWTToken" - }, "InvalidOrExpiredProjectToken": { "properties": { "message": { @@ -2479,7 +2788,9 @@ } }, "type": "object", - "required": ["data"], + "required": [ + "data" + ], "title": "MultiFileContextUpdatePostResponse" }, "NoAccessToGetQueryError": { @@ -2504,14 +2815,24 @@ "type": "object", "title": "NoAccessToProvideFeedbackError" }, - "Provider": { - "type": "string", - "enum": ["no_provider", "google"], - "title": "Provider" + "ProjectNotFoundError": { + "properties": { + "message": { + "type": "string", + "title": "Message", + "default": "Project not found. You may need to create or activate it again." + } + }, + "type": "object", + "title": "ProjectNotFoundError" }, "QueryChatMessageRole": { "type": "string", - "enum": ["user", "assistant", "system"], + "enum": [ + "user", + "assistant", + "system" + ], "title": "QueryChatMessageRole" }, "QueryNotFoundError": { @@ -2528,7 +2849,9 @@ "RequestChatCompletion": { "properties": { "model_ids": { - "items": { "type": "integer" }, + "items": { + "type": "integer" + }, "type": "array", "title": "Model Ids", "description": "Models to use for chat completion" @@ -2542,8 +2865,12 @@ "messages": { "items": { "prefixItems": [ - { "$ref": "#/components/schemas/QueryChatMessageRole" }, - { "type": "string" } + { + "$ref": "#/components/schemas/QueryChatMessageRole" + }, + { + "type": "string" + } ], "type": "array", "maxItems": 2, @@ -2558,7 +2885,14 @@ "description": "Context data for completion" }, "store_context": { - "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Store Context", "description": "Whether to store context data in database", "default": false @@ -2568,7 +2902,14 @@ "description": "Contextual telemetry data" }, "store_contextual_telemetry": { - "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Store Contextual Telemetry", "description": "Whether to store contextual telemetry in database", "default": true @@ -2578,13 +2919,27 @@ "description": "Behavioral telemetry data" }, "store_behavioral_telemetry": { - "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Store Behavioral Telemetry", "description": "Whether to store behavioral telemetry in database", "default": true }, "web_enabled": { - "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Web Enabled", "description": "Whether web access is enabled", "default": false @@ -2604,7 +2959,9 @@ "RequestCompletion": { "properties": { "model_ids": { - "items": { "type": "integer" }, + "items": { + "type": "integer" + }, "type": "array", "title": "Model Ids", "description": "Models to use for completion" @@ -2614,7 +2971,14 @@ "description": "Context data for completion" }, "store_context": { - "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Store Context", "description": "Whether to store context data in database", "default": false @@ -2624,7 +2988,14 @@ "description": "Contextual telemetry data" }, "store_contextual_telemetry": { - "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Store Contextual Telemetry", "description": "Whether to store contextual telemetry in database", "default": true @@ -2634,15 +3005,29 @@ "description": "Behavioral telemetry data" }, "store_behavioral_telemetry": { - "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Store Behavioral Telemetry", "description": "Whether to store behavioral telemetry in database", "default": true }, "stop_sequences": { "anyOf": [ - { "items": { "type": "string" }, "type": "array" }, - { "type": "null" } + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } ], "title": "Stop Sequences", "description": "List of sequences that signal the end of generation", @@ -2702,12 +3087,12 @@ "type": "integer", "minimum": 0.0, "title": "Generation Time", - "description": "Generation time" + "description": "Generation time in ms" }, "confidence": { "type": "number", "title": "Confidence", - "description": "Confidence score" + "description": "Confidence score of the result" } }, "type": "object", @@ -2718,7 +3103,8 @@ "generation_time", "confidence" ], - "title": "ResponseCompletionItem" + "title": "ResponseCompletionItem", + "description": "Represents a successful model completion result." }, "ResponseCompletionResponseData": { "properties": { @@ -2731,18 +3117,26 @@ "completions": { "items": { "anyOf": [ - { "$ref": "#/components/schemas/ResponseCompletionItem" }, - { "$ref": "#/components/schemas/CompletionErrorItem" } + { + "$ref": "#/components/schemas/ResponseCompletionItem" + }, + { + "$ref": "#/components/schemas/CompletionErrorItem" + } ] }, "type": "array", "title": "Completions", - "description": "Generated completions" + "description": "List of generated completions or errors" } }, "type": "object", - "required": ["meta_query_id", "completions"], - "title": "ResponseCompletionResponseData" + "required": [ + "meta_query_id", + "completions" + ], + "title": "ResponseCompletionResponseData", + "description": "Container for completion responses for a given meta-query." }, "ResponseFeedbackResponseData": { "properties": { @@ -2760,8 +3154,12 @@ } }, "type": "object", - "required": ["meta_query_id", "model_id"], - "title": "ResponseFeedbackResponseData" + "required": [ + "meta_query_id", + "model_id" + ], + "title": "ResponseFeedbackResponseData", + "description": "Response model for storing feedback about completions." }, "ResponseUser": { "properties": { @@ -2784,7 +3182,7 @@ "type": "string", "format": "password", "title": "Password", - "description": "User's password (will be hashed)", + "description": "User's password (hashed)", "writeOnly": true }, "config_id": { @@ -2797,35 +3195,57 @@ "type": "string", "format": "uuid", "title": "User Id", - "description": "Unique id for the user" + "description": "Unique ID for the user" }, "joined_at": { "type": "string", "format": "date-time", "title": "Joined At", - "description": "When the user was created" + "description": "Timestamp when the user joined" }, "verified": { "type": "boolean", "title": "Verified", - "description": "Whether the user's email has been verified" + "description": "Whether the user's email is verified" }, "preference": { "anyOf": [ - { "additionalProperties": true, "type": "object" }, - { "type": "null" } + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } ], "title": "Preference", - "description": "Users preference for data management", + "description": "User preferences for data management", "default": {} }, "auth_token": { "anyOf": [ - { "type": "string", "format": "uuid" }, - { "type": "null" } + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } ], "title": "Auth Token", - "description": "Last auth token used to login" + "description": "Last authentication token used by the user" + }, + "is_admin": { + "type": "boolean", + "title": "Is Admin", + "description": "Whether the user is an admin", + "default": false + }, + "can_research": { + "type": "boolean", + "title": "Can Research", + "description": "Whether an admin enabled this account for research", + "default": false } }, "type": "object", @@ -2838,7 +3258,8 @@ "joined_at", "verified" ], - "title": "ResponseUser" + "title": "ResponseUser", + "description": "Response model representing a user in the system." }, "RetrieveChatCompletionsError": { "properties": { @@ -2862,11 +3283,24 @@ "type": "object", "title": "RetrieveCompletionsError" }, + "SessionNotFoundError": { + "properties": { + "message": { + "type": "string", + "title": "Message", + "default": "Session not found. You may need to log in again." + } + }, + "type": "object", + "title": "SessionNotFoundError" + }, "UpdateMultiFileContext": { "properties": { "context_updates": { "additionalProperties": { - "items": { "$ref": "#/components/schemas/FileContextChangeData" }, + "items": { + "$ref": "#/components/schemas/FileContextChangeData" + }, "type": "array" }, "type": "object", @@ -2875,61 +3309,103 @@ } }, "type": "object", - "required": ["context_updates"], + "required": [ + "context_updates" + ], "title": "UpdateMultiFileContext" }, "UpdateUser": { "properties": { "name": { "anyOf": [ - { "type": "string", "maxLength": 50, "minLength": 3 }, - { "type": "null" } + { + "type": "string", + "maxLength": 50, + "minLength": 3 + }, + { + "type": "null" + } ], "title": "Name", "description": "User's new name" }, "email": { "anyOf": [ - { "type": "string", "format": "email" }, - { "type": "null" } + { + "type": "string", + "format": "email" + }, + { + "type": "null" + } ], "title": "Email", "description": "New email of user" }, "previous_password": { "anyOf": [ - { "type": "string", "format": "password", "writeOnly": true }, - { "type": "null" } + { + "type": "string", + "format": "password", + "writeOnly": true + }, + { + "type": "null" + } ], "title": "Previous Password", "description": "Previous password of user" }, "password": { "anyOf": [ - { "type": "string", "format": "password", "writeOnly": true }, - { "type": "null" } + { + "type": "string", + "format": "password", + "writeOnly": true + }, + { + "type": "null" + } ], "title": "Password", "description": "New password of user" }, "preference": { "anyOf": [ - { "additionalProperties": true, "type": "object" }, - { "type": "null" } + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } ], "title": "Preference", "description": "Updated user preferences as a dictionary" }, "config_id": { "anyOf": [ - { "type": "integer", "minimum": 1.0 }, - { "type": "null" } + { + "type": "integer", + "minimum": 1.0 + }, + { + "type": "null" + } ], "title": "Config Id", "description": "New configuration ID" }, "verified": { - "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Verified", "description": "Whether user verified or not" } @@ -2961,7 +3437,9 @@ } }, "type": "object", - "required": ["user"], + "required": [ + "user" + ], "title": "UpdateUserPutResponse" }, "UserAlreadyExistsWithThisEmail": { @@ -2997,7 +3475,7 @@ "type": "object", "title": "VerifyUserError" }, - "VerifyUserPostHTMLResponse": { + "VerifyUserGetHTMLResponse": { "properties": { "message": { "type": "string", @@ -3007,11 +3485,11 @@ "html": { "type": "string", "title": "Html", - "default": "\n \n \n Email Verification Successful\n \n \n \n
Email Verification Successful!
\n
Your email has been verified successfully. You can now close this page and continue using the application.
\n Go to Homepage\n \n \n " + "default": "\n\n\n Email Verification Successful\n \n\n\n
Email Verification Successful!
\n
Your email has been verified successfully. You can now close this page and continue using the application.
\n Go to Homepage\n\n" } }, "type": "object", - "title": "VerifyUserPostHTMLResponse" + "title": "VerifyUserGetHTMLResponse" } } } diff --git a/packaging/README.md b/packaging/README.md new file mode 100644 index 0000000..447ed26 --- /dev/null +++ b/packaging/README.md @@ -0,0 +1,59 @@ +# Managed runtime packaging + +Build on each target operating system and architecture; PyInstaller does not +cross-compile. From the repository root: + +```bash +python -m pip install -r packaging/requirements-runtime.lock -r packaging/requirements-build.lock +python -m pip install -r packaging/requirements-test.lock +python -m pip install --no-deps . +PYTHONPATH=src python -m research.study.agents.participant_release native \ + --version 1.2.0 --platform macos-arm64 \ + --server-commit "$(git rev-parse HEAD)" --output dist/release-1.2.0 +``` + +The version is embedded in the executable and must match the immutable runtime +release tag. The workflow performs this stamp automatically; +reset `_build_version.py` to its source-development value after a manual build. + +`requirements-runtime.lock` pins the complete participant dependency graph. +Regenerate it with the command recorded in the lock file and review all changes +before publishing a study release. + +The one-folder artifact is `dist/code4me2-agent/` (the executable is suffixed +`.exe` on Windows). Archive the directory *contents* as a flat zip: the +executable must sit at the archive root (`code4me2-agent` / +`code4me2-agent.exe`), with `_internal/` beside it. Materialize macOS +framework symlinks as real files under their link paths (the plugin installer +extracts with `ZipInputStream` and does not restore symlinks); the extracted +copy must still pass `--self-check`. The runtime release manifest +(`code4me-managed-runtime-release.json`) supplies each archive's version, target, +SHA-256 digest, size and download URL; the plugin downloads the archive a study +pins and verifies it. The `build-managed-runtime` workflow implements this layout; +do not revert to archiving the parent folder. + +Code signing is optional. The plugin downloads the archive itself, verifies the +SHA-256 pinned by the study server and launches the executable directly, so +macOS Gatekeeper (which only assesses quarantined downloads) and Windows +SmartScreen (which only checks Mark-of-the-Web files) never evaluate it. The +pin, not a signature, is the integrity control. Signing mainly helps Windows +hosts with Smart App Control or strict antivirus. To sign, dispatch the +`build-managed-runtime` workflow with `sign: true`: macOS binaries are signed +and the archive notarized, Windows binaries Authenticode-signed, in their native +jobs before the archive is hashed and tested. This needs the `CODE4ME_MACOS_*`, +`CODE4ME_APPLE_*` and `CODE4ME_WINDOWS_*` secrets named in the workflow; a +requested signing run fails when they are missing. Publicly trusted Windows +code-signing keys must live in hardware since June 2023, so the PFX secret only +works for a legacy exportable certificate. Signing changes the archive bytes, so +a signed build is a new runtime version with its own SHA-256. + +## Deferred managed runtimes + +- TODO: package, register, repair and certify Goose without PATH discovery. +- TODO: package, register, repair and certify Codex without Node/npm or a source checkout. +- TODO: add both runtimes to shared onboarding, policy enforcement and telemetry tests. + +Existing Goose and Codex developer integrations remain supported while these +participant distribution tasks are pending. + +See [release import and plugin integration](../docs/research-platform/RELEASES.md). diff --git a/packaging/archive_runtime.py b/packaging/archive_runtime.py new file mode 100644 index 0000000..0b1c270 --- /dev/null +++ b/packaging/archive_runtime.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Create the flat, symlink-free runtime ZIP consumed by the IDE plugin.""" + +from __future__ import annotations + +import argparse +import shutil +import stat +import zipfile +from pathlib import Path + + +def collect_tree(root: Path) -> dict[str, Path]: + entries: dict[str, Path] = {} + root_real = root.resolve() + + def visit(directory: Path, archive_prefix: Path, ancestors: frozenset[Path], external_tree: bool = False) -> None: + real_directory = directory.resolve() + if real_directory in ancestors: + return + next_ancestors = ancestors | {real_directory} + for child in sorted(directory.iterdir(), key=lambda item: item.name): + archive_path = (archive_prefix / child.name).as_posix() + resolved_child = child.resolve() + if not resolved_child.is_relative_to(root_real) and not external_tree: + if child.is_symlink() and child.is_dir(): + visit(resolved_child, archive_prefix / child.name, next_ancestors, True) + continue + if child.is_symlink() and child.is_file(): + entries.setdefault(archive_path, resolved_child) + continue + raise SystemExit( + f"runtime bundle contains an unsupported external entry: {child}" + ) + if child.is_dir(): + visit(child, archive_prefix / child.name, next_ancestors, external_tree) + elif child.is_file(): + entries.setdefault(archive_path, resolved_child) + + visit(root, Path(), frozenset()) + return entries + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path("dist/code4me2-agent")) + parser.add_argument("--platform", required=True) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + root = args.root.resolve() + if not root.is_dir(): + raise SystemExit(f"runtime bundle does not exist: {root}") + executable = "code4me2-agent.exe" if args.platform.startswith("windows") else "code4me2-agent" + entries = collect_tree(root) + if executable not in entries: + raise SystemExit(f"runtime executable is missing at archive root: {executable}") + + output = args.output or Path(f"code4me-agent-{args.platform}.zip") + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as archive: + for archive_path, source in sorted(entries.items()): + info = zipfile.ZipInfo(archive_path) + mode = 0o755 if archive_path == executable else 0o644 + info.external_attr = (stat.S_IFREG | mode) << 16 + info.create_system = 3 + info.compress_type = zipfile.ZIP_DEFLATED + with source.open("rb") as source_handle, archive.open(info, "w") as target_handle: + shutil.copyfileobj(source_handle, target_handle, length=1024 * 1024) + print(f"archived {output} ({len(entries)} files)") + + +if __name__ == "__main__": + main() diff --git a/packaging/code4me2-agent.spec b/packaging/code4me2-agent.spec new file mode 100644 index 0000000..507f794 --- /dev/null +++ b/packaging/code4me2-agent.spec @@ -0,0 +1,37 @@ +from pathlib import Path + +# SPECPATH is the directory containing this spec file (packaging/), +# so its parent is the server repository root. +project_root = Path(SPECPATH).parent +source_root = project_root / "src" + +analysis = Analysis( + [str(source_root / "code4me2_agent" / "cli.py")], + pathex=[str(source_root)], + binaries=[], + datas=[], + hiddenimports=[], + excludes=["backend", "database", "celery", "torch", "transformers"], + noarchive=False, +) + +pyz = PYZ(analysis.pure) + +executable = EXE( + pyz, + analysis.scripts, + [], + exclude_binaries=True, + name="code4me2-agent", + console=True, + disable_windowed_traceback=False, +) + +collection = COLLECT( + executable, + analysis.binaries, + analysis.datas, + strip=False, + upx=False, + name="code4me2-agent", +) diff --git a/packaging/requirements-build.lock b/packaging/requirements-build.lock new file mode 100644 index 0000000..771c720 --- /dev/null +++ b/packaging/requirements-build.lock @@ -0,0 +1,5 @@ +# Install the project itself first so its runtime dependencies are resolved: +# python -m pip install . -r packaging/requirements-build.lock +pyinstaller==6.16.0 +setuptools==80.9.0 +wheel==0.45.1 diff --git a/packaging/requirements-runtime.lock b/packaging/requirements-runtime.lock new file mode 100644 index 0000000..fbb13c7 --- /dev/null +++ b/packaging/requirements-runtime.lock @@ -0,0 +1,114 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml --python-version 3.13 --output-file packaging/requirements-runtime.lock +agent-client-protocol==0.12.1 + # via code4me2-agent (pyproject.toml) +annotated-types==0.8.0 + # via pydantic +anyio==4.15.1 + # via + # httpx + # httpx2 + # mcp + # openai + # sse-starlette + # starlette +attrs==26.1.0 + # via + # jsonschema + # referencing +certifi==2026.7.22 + # via + # httpcore + # httpx +cffi==2.1.1 + # via cryptography +click==8.5.0 + # via uvicorn +cryptography==50.0.1 + # via pyjwt +distro==1.9.0 + # via openai +h11==0.16.0 + # via + # httpcore + # httpcore2 + # uvicorn +httpcore==1.0.9 + # via httpx +httpcore2==2.12.0 + # via httpx2 +httpx==0.28.1 + # via openai +httpx2==2.12.0 + # via mcp +idna==3.19 + # via + # anyio + # httpx + # httpx2 +jiter==0.16.0 + # via openai +jsonschema==4.26.0 + # via mcp +jsonschema-specifications==2025.9.1 + # via jsonschema +mcp==2.2.0 + # via code4me2-agent (pyproject.toml) +mcp-types==2.2.0 + # via mcp +openai==1.109.1 + # via code4me2-agent (pyproject.toml) +opentelemetry-api==1.44.0 + # via mcp +pycparser==3.0 + # via cffi +pydantic==2.13.5 + # via + # agent-client-protocol + # mcp + # mcp-types + # openai +pydantic-core==2.46.5 + # via pydantic +pyjwt==2.13.0 + # via mcp +python-multipart==0.0.32 + # via mcp +referencing==0.37.0 + # via + # jsonschema + # jsonschema-specifications +rpds-py==2026.6.3 + # via + # jsonschema + # referencing +sniffio==1.3.1 + # via openai +sse-starlette==3.4.11 + # via mcp +starlette==1.6.0 + # via + # mcp + # sse-starlette +tqdm==4.70.0 + # via openai +truststore==0.10.4 + # via + # httpcore2 + # httpx2 +typing-extensions==4.16.0 + # via + # anyio + # mcp + # mcp-types + # openai + # opentelemetry-api + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.4 + # via + # mcp + # pydantic +uvicorn==0.52.4 + # via mcp diff --git a/packaging/requirements-test.lock b/packaging/requirements-test.lock new file mode 100644 index 0000000..35cd878 --- /dev/null +++ b/packaging/requirements-test.lock @@ -0,0 +1,8 @@ +# Runtime release smoke-test dependencies. Keep fully pinned for native builds. +colorama==0.4.6 ; sys_platform == "win32" +iniconfig==2.3.0 +packaging==26.3 +pluggy==1.6.0 +pygments==2.21.0 +pytest==9.1.1 +pytest-env==1.1.5 diff --git a/packaging/stamp_runtime_version.py b/packaging/stamp_runtime_version.py new file mode 100644 index 0000000..4e4a7b7 --- /dev/null +++ b/packaging/stamp_runtime_version.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Stamp the immutable version reported by a packaged runtime binary.""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +SEMVER = re.compile( + r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?\Z" +) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--version", required=True) + parser.add_argument( + "--output", + type=Path, + default=Path("src/code4me2_agent/_build_version.py"), + ) + args = parser.parse_args() + if not SEMVER.fullmatch(args.version): + raise SystemExit(f"runtime version must be SemVer: {args.version!r}") + args.output.write_text( + '"""Generated by packaging/stamp_runtime_version.py."""\n\n' + f'RUNTIME_VERSION = "{args.version}"\n', + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/populate_test_data.py b/populate_test_data.py index ac61a57..88ce17d 100644 --- a/populate_test_data.py +++ b/populate_test_data.py @@ -5,11 +5,10 @@ This script populates the database with realistic test data to demonstrate the analytics and visualization features including: - Users (admins and regular users) -- Multiple configurations for A/B testing +- Multiple configurations for testing - Realistic usage patterns over time - Model performance with varying acceptance rates - Programming language distribution -- Studies and configuration assignments - Telemetry data Run this script to generate test data for the analytics dashboard. @@ -106,7 +105,7 @@ } ] -# Different configurations for A/B testing +# Different completion configurations for the analytics dashboards CONFIGS_FOR_TESTING = [ { "name": "Default Configuration", @@ -160,8 +159,7 @@ def __init__(self): 'trigger_types': [], 'plugin_versions': [], 'projects': [], - 'sessions': [], - 'studies': [] + 'sessions': [] } def connect(self): @@ -192,8 +190,6 @@ def clear_existing_data(self): 'chat_query', 'completion_query', 'meta_query', - 'config_assignment_history', - 'study', 'session_projects', 'session', 'project_users', @@ -730,77 +726,6 @@ def _generate_ground_truth(self, language_name: str) -> str: """Generate ground truth data""" return f"// Correct implementation for {language_name}\n" + self._generate_code_completion(language_name) - def create_studies(self): - """Create test studies for A/B testing""" - print("🔬 Creating test studies...") - - # Create a few studies to demonstrate the feature - studies_to_create = [ - { - "name": "Model Performance Comparison", - "description": "Testing different model selection strategies", - "config_ids": [1, 2, 3], # Use first 3 configs - "is_active": True, - "days_ago_started": 7 - }, - { - "name": "Response Time Optimization", - "description": "Comparing fast vs quality-optimized responses", - "config_ids": [2, 3], # Fast vs High Quality - "is_active": False, - "days_ago_started": 20, - "days_ago_ended": 10 - } - ] - - for study_info in studies_to_create: - study_id = str(uuid.uuid4()) - admin_user = random.choice([u for u in self.created_data['users'] if u['is_admin']]) - - starts_at = datetime.now() - timedelta(days=study_info['days_ago_started']) - ends_at = None - if 'days_ago_ended' in study_info: - ends_at = datetime.now() - timedelta(days=study_info['days_ago_ended']) - - # Use first config as default - default_config_id = self.created_data['configs'][0]['config_id'] - - self.cursor.execute(''' - INSERT INTO study (study_id, name, description, created_by, starts_at, - ends_at, is_active, default_config_id, created_at) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) - ''', ( - study_id, study_info['name'], study_info['description'], - admin_user['user_id'], starts_at, ends_at, study_info['is_active'], - default_config_id, starts_at - )) - - # Assign users to configs for this study - study_configs = [c for c in self.created_data['configs'] - if c['config_id'] in study_info['config_ids']] - - for user in self.created_data['users']: - if not user['is_admin']: # Only assign regular users to studies - assigned_config = random.choice(study_configs) if study_configs else random.choice(self.created_data['configs']) - self.cursor.execute(''' - INSERT INTO config_assignment_history - (user_id, study_id, assigned_config_id, assigned_at) - VALUES (%s, %s, %s, %s) - ''', ( - user['user_id'], study_id, assigned_config['config_id'], starts_at - )) - - self.created_data['studies'].append({ - 'study_id': study_id, - 'name': study_info['name'], - 'is_active': study_info['is_active'] - }) - - print(f" Created study: {study_info['name']}") - - print(f"✅ Created {len(studies_to_create)} studies") - self.conn.commit() - def print_summary(self): """Print summary of created data""" print("\n🎉 Test Data Population Complete!") @@ -811,7 +736,6 @@ def print_summary(self): print(f" • Projects: {len(self.created_data['projects'])}") print(f" • Sessions: {len(self.created_data['sessions'])}") print(f" • Queries: {NUM_QUERIES}") - print(f" • Studies: {len(self.created_data['studies'])}") print(f" • Models: {len(self.created_data['models'])}") print(f" • Languages: {len(self.created_data['languages'])}") @@ -819,17 +743,11 @@ def print_summary(self): for user in self.created_data['users']: if user['is_admin']: print(f" • {user['email']} (Config ID: {user['config_id']})") - - print(f"\n🔬 Active Studies:") - for study in self.created_data['studies']: - if study['is_active']: - print(f" • {study['name']}") print(f"\n🌐 You can now:") print(f" • Login with any admin user to see all analytics") print(f" • Login with regular users to see personal analytics") print(f" • Test the dashboard time windows and filters") - print(f" • View A/B test results in admin panel") print(f" • Explore model performance comparisons") @@ -854,7 +772,6 @@ def main(): populator.create_projects() populator.create_sessions() populator.create_telemetry_and_queries() - populator.create_studies() # Print summary populator.print_summary() diff --git a/pyproject.toml b/pyproject.toml index e54d76c..bb3175d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,38 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "code4me2-agent" +version = "0.1.0" +description = "Code4Me ACP agent runtime" +requires-python = ">=3.10" +dependencies = [ + # Pre-1.0 SDK: allow bugfix releases in the 0.12 series. Compatibility is + # covered by the ACP runtime tests; minor bumps need a re-run, not a migration. + "agent-client-protocol~=0.12.1", + # ACP requires agents to support client-supplied stdio MCP servers. + "mcp==2.2.0", + # mcp>=3.0.0 would otherwise pull sse-starlette>=3.0.3, which requires + # starlette>=0.49.1 and conflicts with the backend's fastapi~=0.115.0 + # (starlette<0.47). 3.0.0 is the last release without that hard floor. + "sse-starlette>=3.0.0,<3.0.3", + "openai>=1.0,<2", + # Trust roots for the agent's own HTTPS calls; the packaged runtime has no + # usable platform CA store on macOS (code4me2_agent.tls). + "certifi", +] + +[project.scripts] +code4me2-agent = "code4me2_agent.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["code4me2_agent*"] + [tool.ruff] line-length = 99 [tool.ruff.lint] select = ["E", "F", "W", "Q", "I", "TCH"] -ignore = ["E203", "E402", "E501", "F841", "F401", "F811"] \ No newline at end of file +ignore = ["E203", "E402", "E501", "F841", "F401", "F811"] diff --git a/pytest.ini b/pytest.ini index 59a7237..d3526fb 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,4 +3,5 @@ pythonpath= src testpaths = tests env = TEST_MODE=true + BOOTSTRAP_SIGNING_SECRET=test-bootstrap-signing-secret ;addopts = --ignore=src/database/migration/versions \ No newline at end of file diff --git a/redis.prod.conf b/redis.prod.conf new file mode 100644 index 0000000..87e6834 --- /dev/null +++ b/redis.prod.conf @@ -0,0 +1,16 @@ +# Production Redis for auth/session tokens and ACP grants (production-readiness B-05). +# +# Everything that keeps a participant signed in and an agent authorized lives in +# this instance, so it must survive a container restart: AOF with a per-second +# fsync on the `redis_data` volume. The password is passed on the command line by +# docker-compose.prod.yml (`--requirepass "$REDIS_PASSWORD"`) so this file holds +# no secret. Keep `notify-keyspace-events Ex`: the backend subscribes to key +# expiry events to end sessions. +appendonly yes +appendfsync everysec +save 900 1 +save 300 100 +dir /data +notify-keyspace-events Ex +maxmemory-policy noeviction +protected-mode yes diff --git a/requirements.txt b/requirements.txt index b62198b..7ce4cd2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,9 @@ pytest-mock~=3.10.0 pytest-postgresql~=6.0.0 pytest-env~=1.1.0 hypothesis~=6.70.0 +# Used by tests/backend_tests/research/test_telemetry.py to validate the emitted +# telemetry JSON Schema (Draft 2020-12) directly. +jsonschema~=4.26.0 # ----------------------- # Core Backend @@ -28,7 +31,7 @@ detect_secrets~=1.5.0 starlette~=0.46.0 celery~=5.5.0 httpx~=0.25.0 -pydantic~=2.11.3 +pydantic>=2.11,<3 pydantic_settings~=2.9.1 polyfactory~=2.21.0 fastapi-limiter~=0.1.6 @@ -41,7 +44,9 @@ python-multipart~=0.0.20 # Database # ----------------------- SQLAlchemy~=2.0.30 -psycopg_binary~=3.2.1 +# Install psycopg and its binary implementation as one version-locked extra. +# Separate requirements allow pip to select incompatible minor versions. +psycopg[binary]~=3.2.1 psycopg2-binary~=2.9.9 alembic>=1.13.1 @@ -53,7 +58,6 @@ numpy>=1.26.2 # ----------------------- # Auth & Security # ----------------------- -google-auth~=2.22.0 # ----------------------- # LLMs & AI @@ -63,7 +67,7 @@ langchain-community~=0.3.24 langchain-huggingface~=0.2.0 transformers~=4.50.0 accelerate~=1.7.0 -torch~=2.7.0 # Uncomment if using PyTorch +torch>=2.7,<3 # required by sentence-transformers (CPU/MPS wheels) # vllm~=0.5.0 # Uncomment if using vLLM huggingface_hub[cli] @@ -71,3 +75,9 @@ huggingface_hub[cli] # Utilities # ----------------------- requests~=2.32.3 + +# ----------------------- +# Research exports (Issue 12) +# ----------------------- +# Canonical research export artifact writer/reader. The export service degrades +# to CSV/JSONL with a typed EXPORT_WRITER_UNAVAILABLE when this is absent. diff --git a/scripts/dev/export_openapi.py b/scripts/dev/export_openapi.py new file mode 100644 index 0000000..d1ed55e --- /dev/null +++ b/scripts/dev/export_openapi.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Export the curated plugin-facing OpenAPI snapshot from the FastAPI routers. + +The tracked ``openapi.json`` is **not** a full API dump: it is a curated +18-path snapshot of the plugin-facing routes. Before this script the generation +invocation was uncommitted, so the snapshot drifted from the source. This +exporter is that invocation, and it is deterministic: + +1. build the route table without the application config/database (the schema + only depends on the routers and their Pydantic models); +2. keep exactly the curated paths (``CURATED_PATHS``); +3. prune ``components`` to the schemas transitively referenced by those paths; +4. preserve the path order already on disk so a refresh produces a reviewable + diff; +5. write the pretty server snapshot. + +Usage:: + + python scripts/dev/export_openapi.py # refresh openapi.json + python scripts/dev/export_openapi.py --check # fail if stale (the backend suite runs this) + +This exporter owns the **server** snapshot only. The IntelliJ plugin's bundled +client contract (`code4me2/src/main/resources/backend/api/openapi.json`) is a +different, larger artifact (22 paths, including session deactivation, user +lookup and password reset) that is the input for the generated Kotlin client. +Do not copy this snapshot over it: doing so silently drops client paths. The +Kotlin client in `code4me2/generated` is regenerated separately; this script +prints the exact ``openapi-generator-cli`` invocation to use. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Iterator + +# The curated plugin-facing snapshot. Adding a path here is a deliberate +# snapshot change, never an automatic consequence of adding a server route. +# (This is not the plugin's bundled client contract; see the module docstring.) +CURATED_PATHS: tuple[str, ...] = ( + "/api/session/acquire", + "/api/project/create", + "/api/project/activate", + "/api/user/create", + "/api/user/update", + "/api/user/delete", + "/api/user/authenticate", + "/api/user/verify/check", + "/api/user/verify/resend", + "/api/user/verify/", + "/api/completion/request", + "/api/completion/feedback", + "/api/completion/{query_id}", + "/api/completion/multi-file-context/update", + "/api/chat/request", + "/api/chat/get/{page_number}", + "/api/chat/delete/{chat_id}", + "/api/ping", +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +SRC_ROOT = REPO_ROOT / "src" + + +def _build_schema() -> dict[str, Any]: + """Build the OpenAPI schema without importing the configured application.""" + sys.path.insert(0, str(SRC_ROOT)) + from fastapi import FastAPI # noqa: PLC0415 - import after sys.path is set + + from backend.routers import router # noqa: PLC0415 + + app = FastAPI( + title="Code4Me V2 API", + description="The complete API for Code4Me V2", + version="1.0.0", + ) + app.include_router(router, prefix="/api") + return app.openapi() + + +def _refs(node: Any) -> Iterator[str]: + """Yield every ``#/components/...`` reference under ``node``.""" + if isinstance(node, dict): + for key, value in node.items(): + if key == "$ref" and isinstance(value, str) and value.startswith("#/components/"): + yield value + else: + yield from _refs(value) + elif isinstance(node, list): + for item in node: + yield from _refs(item) + + +def _prune_components(spec: dict[str, Any]) -> None: + """Keep only the components transitively referenced by the kept paths.""" + components = spec.get("components", {}) + used: dict[str, set[str]] = {} + + def record(ref: str) -> bool: + parts = ref.split("/") + if len(parts) < 4: + return False + section, name = parts[2], "/".join(parts[3:]) + names = used.setdefault(section, set()) + if name in names: + return False + names.add(name) + return True + + for ref in _refs(spec.get("paths", {})): + record(ref) + + changed = True + while changed: + changed = False + for section, names in list(used.items()): + for name in list(names): + for ref in _refs(components.get(section, {}).get(name)): + changed = record(ref) or changed + + pruned: dict[str, Any] = {} + for section, entries in components.items(): + kept = {name: node for name, node in entries.items() if name in used.get(section, set())} + if kept: + pruned[section] = kept + spec["components"] = pruned + + +def _existing_path_order(output: Path) -> list[str]: + if not output.is_file(): + return [] + try: + current = json.loads(output.read_text(encoding="utf-8")) + except (ValueError, OSError): + return [] + return list(current.get("paths", {})) + + +def curated_spec(output: Path) -> dict[str, Any]: + """The deterministic curated contract for ``output`` (used for ordering).""" + spec = _build_schema() + paths = spec.get("paths", {}) + missing = [path for path in CURATED_PATHS if path not in paths] + if missing: + raise SystemExit( + "the server no longer exposes curated client path(s): " + ", ".join(missing) + ) + # Preserve the on-disk order so a refresh diff only shows real changes. + order = [path for path in _existing_path_order(output) if path in CURATED_PATHS] + order += [path for path in CURATED_PATHS if path not in order] + spec["paths"] = {path: paths[path] for path in order} + _prune_components(spec) + return spec + + +def _render(spec: dict[str, Any]) -> str: + return json.dumps(spec, indent=2, ensure_ascii=False) + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output", + type=Path, + default=REPO_ROOT / "openapi.json", + help="server snapshot to write (default: /openapi.json)", + ) + parser.add_argument( + "--check", + action="store_true", + help="fail if the tracked snapshot is not identical to the export", + ) + args = parser.parse_args() + + spec = curated_spec(args.output) + rendered = _render(spec) + + if args.check: + current = args.output.read_text(encoding="utf-8") if args.output.is_file() else "" + if current != rendered: + print(f"{args.output} is stale; run scripts/dev/export_openapi.py", file=sys.stderr) + return 1 + print(f"{args.output} is up to date ({len(spec['paths'])} paths)") + return 0 + + args.output.write_text(rendered, encoding="utf-8") + print(f"wrote {args.output} ({len(spec['paths'])} paths, {len(spec['components'].get('schemas', {}))} schemas)") + + print( + "Kotlin client (code4me2/generated) is regenerated separately:\n" + " cd code4me2 && npx --yes @openapitools/openapi-generator-cli@7.13.0 generate \\\n" + " -i src/main/resources/backend/api/openapi.json -g kotlin \\\n" + " -o generated --additional-properties=library=jvm-okhttp4,serializationLibrary=moshi" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/dev/forbidden_symbols.txt b/scripts/dev/forbidden_symbols.txt new file mode 100644 index 0000000..b5f8e5d --- /dev/null +++ b/scripts/dev/forbidden_symbols.txt @@ -0,0 +1,16 @@ +# Forbidden-symbol allowlist for scripts/dev/scan_forbidden_symbols.py. +# +# Format: | # +# A per-symbol entry only covers the named symbol, so a *different* forbidden +# concept reappearing in an allowlisted file still fails the scan. +# +# Every entry is a temporary retirement record and must name an owner and a +# planned-removal review trigger. The condition_id decision (authoring-only +# vocabulary, no execution consumer) is recorded in +# docs/PROTOCOL_VOCABULARY_DECISION.md. Review all entries at the next protocol +# cleanup after ISSUE-19: delete any symbol whose authoring module has no +# remaining consumer, otherwise re-justify it with a fresh owner and trigger. + +code4me2-server/src/research/study/protocol/models.py | condition_id # authoring-only StudyCondition field; owner: server protocol maintainers; removal review: ISSUE-19 follow-up when the protocol-authoring model is ported or removed +code4me2-server/src/research/study/protocol/validation.py | condition_id # condition uniqueness/weights for protocol authoring only; owner: server protocol maintainers; removal review: ISSUE-19 follow-up once no current validation consumer exists +code4me2-server/src/research/study/protocol/canonical.py | condition_id # canonical hashing of the authoring condition list; owner: server protocol maintainers; removal review: ISSUE-19 follow-up when protocol digesting no longer includes conditions diff --git a/scripts/dev/generate_canonical_event_schema.py b/scripts/dev/generate_canonical_event_schema.py new file mode 100644 index 0000000..f7e6e9e --- /dev/null +++ b/scripts/dev/generate_canonical_event_schema.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Regenerate the canonical telemetry JSON Schema from the Python model. + +``CanonicalEventV1`` (``research.telemetry.models``) is the single source of +truth for the event envelope; the closed vocabularies come from +``research.telemetry.enums``. This script writes the language-neutral contract +consumed by generated clients. + +It is intentionally dependency-free and side-effect limited to the schema file. +The test suite asserts that the checked-in file equals the generated output, so a +model change that is not accompanied by a regeneration fails fast. + +Usage (from ``code4me2-server/``):: + + python scripts/dev/generate_canonical_event_schema.py +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +SRC = PROJECT_ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from research.telemetry.schema import ( # noqa: E402 + CANONICAL_EVENT_V1_SCHEMA_PATH, + write_canonical_event_v1_schema, +) + + +def main() -> int: + path = write_canonical_event_v1_schema(CANONICAL_EVENT_V1_SCHEMA_PATH) + print(f"wrote {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/dev/scan_forbidden_symbols.py b/scripts/dev/scan_forbidden_symbols.py new file mode 100644 index 0000000..f5775a3 --- /dev/null +++ b/scripts/dev/scan_forbidden_symbols.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Forbidden-symbol scan for the research cleanup boundary (Task 06 §13, Task 08 §6). + +Scans active research source (never build output, caches, virtualenvs or +generated clients) for revision/condition concepts that the study-lifecycle +redesign removed. The scan fails when a hit is not covered by the allowlist, so +every remaining match must be an explicitly reviewed exception. + +Usage (from the repository root or anywhere): + + python scripts/dev/scan_forbidden_symbols.py [--workspace /path/to/Code4Me_Paper] + +Exit codes: 0 = clean or allowlisted, 1 = unallowlisted hits, 2 = scan error. + +Allowlist format (``scripts/dev/forbidden_symbols.txt``): one entry per line, +`` # ``. Blank lines and ``#`` comments are +ignored. A hit matches when the file's repo-relative path equals the entry or +starts with it. +""" + +from __future__ import annotations + +import argparse +import pathlib +import re +import sys + +PATTERNS = ( + "study_revision_id", + "StudyRevision", + "supersede_revision", + "publish_draft", + "condition_exposure", + "condition_id", +) + +SKIP_DIRS = { + "build", + "__pycache__", + "node_modules", + ".venv", + "venv", + ".gradle", + "generated", + "dist", + ".git", +} + +SCAN_ROOTS = ( + "code4me2-server/src", + "code4me2/src/main", + "code4me2/telemetry-acp-proxy/telemetry_acp_proxy", +) + +ALLOWLIST = pathlib.Path(__file__).with_name("forbidden_symbols.txt") + + +def _parse_allowlist(path: pathlib.Path) -> list[tuple[str, frozenset[str] | None, str]]: + """Return ``(path_prefix, symbols|None, reason)`` entries. + + Format: `` | # reason``. + Omitting the symbol list (no ``|``) covers every scanned symbol for that path. + A per-symbol entry is preferred: it cannot hide a *different* forbidden + concept reappearing in the same file. + """ + entries: list[tuple[str, frozenset[str] | None, str]] = [] + if not path.exists(): + return entries + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + body, _, reason = line.partition("#") + prefix, _, symbols = body.partition("|") + allowed = ( + frozenset(part.strip() for part in symbols.split(",") if part.strip()) + if symbols.strip() + else None + ) + entries.append((prefix.strip(), allowed, reason.strip())) + return entries + + +def _iter_files(root: pathlib.Path): + if not root.exists(): + return + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + if any(part in SKIP_DIRS for part in path.parts): + continue + if path.suffix in {".pyc", ".class", ".jar", ".png", ".jpg"}: + continue + yield path + + +def _allowed(relative: str, symbol: str, entries) -> str | None: + for prefix, symbols, reason in entries: + if not (relative == prefix or relative.startswith(prefix.rstrip("/") + "/")): + continue + if symbols is not None and symbol not in symbols: + continue + return reason or "allowlisted" + return None + + +def scan(workspace: pathlib.Path) -> int: + entries = _parse_allowlist(ALLOWLIST) + matcher = re.compile("|".join(re.escape(pattern) for pattern in PATTERNS)) + unallowlisted: list[str] = [] + allowed_hits: list[str] = [] + + missing_roots = [root for root in SCAN_ROOTS if not (workspace / root).exists()] + if missing_roots: + print( + "scan error: scan root(s) missing (fail closed): " + + ", ".join(missing_roots), + file=sys.stderr, + ) + return 2 + + for scan_root in SCAN_ROOTS: + root = workspace / scan_root + for path in _iter_files(root): + relative = path.relative_to(workspace).as_posix() + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError as error: # pragma: no cover - unreadable file + print(f"scan error: {relative}: {error}", file=sys.stderr) + return 2 + for number, line in enumerate(text.splitlines(), start=1): + match = matcher.search(line) + if not match: + continue + hit = f"{relative}:{number}: {match.group(0)}: {line.strip()[:160]}" + reason = _allowed(relative, match.group(0), entries) + if reason is None: + unallowlisted.append(hit) + else: + allowed_hits.append(f"{hit} [allowlisted: {reason}]") + + for hit in allowed_hits: + print(f"ALLOWED {hit}") + for hit in unallowlisted: + print(f"FORBIDDEN {hit}", file=sys.stderr) + + if unallowlisted: + print( + f"\n{len(unallowlisted)} forbidden hit(s) outside the allowlist " + f"({ALLOWLIST.name})", + file=sys.stderr, + ) + return 1 + + print(f"\nclean: {len(allowed_hits)} allowlisted hit(s), 0 forbidden hits") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--workspace", + default=str(pathlib.Path(__file__).resolve().parents[2].parent), + help="workspace root containing code4me2/ and code4me2-server/", + ) + args = parser.parse_args() + return scan(pathlib.Path(args.workspace).resolve()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/dev/seed_local_byoa_telemetry_study.py b/scripts/dev/seed_local_byoa_telemetry_study.py new file mode 100644 index 0000000..b4f3e03 --- /dev/null +++ b/scripts/dev/seed_local_byoa_telemetry_study.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 +"""Create one local-only Goose or Codex (BYOA) telemetry test study. + +A development helper for live telemetry checks with a participant-installed +agent. It follows the current research contract: + +* the installed agent must answer ``--version`` and an ACP ``initialize`` + handshake before a release is recorded; +* a Goose release carries the research inference gateway runtime bindings + (the ``participant-release.example.json`` contract), so bootstrap never + blocks with ``INFERENCE_GATEWAY_UNBOUND``; +* Goose arms are metered: the model gets a price on the development provider + connection and the study a default participant budget, because both fail + closed when missing; +* ``--tool-use-test`` enforces the Goose tool selection at the research + inference gateway (Goose reads no tool setting from its environment); +* a Codex arm runs on the participant's own login and is never study-funded. + +It never enrolls or consents a participant: the test account joins through the +normal web flow with the printed join code. + +Usage (from ``code4me2-server/``, development only):: + + CODE4ME_DEV_SEED=1 PYTHONPATH=src python scripts/dev/seed_local_byoa_telemetry_study.py goose + CODE4ME_DEV_SEED=1 PYTHONPATH=src python scripts/dev/seed_local_byoa_telemetry_study.py goose --tool-use-test + CODE4ME_DEV_SEED=1 PYTHONPATH=src python scripts/dev/seed_local_byoa_telemetry_study.py codex + +``TEST_MODE=true`` is accepted in place of ``CODE4ME_DEV_SEED=1``; a database +host other than the local machine is refused. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import queue +import secrets +import shutil +import subprocess +import sys +import threading +import time +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from decimal import Decimal +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence + +REPO = Path(__file__).resolve().parents[2] + +#: Goose 1.51's developer tools as a study gateway call offers them (observed +#: 2026-09-27); the tool-use arm allows these and withholds the rest. +GOOSE_TOOL_USE_SELECTION = ("shell", "edit", "write", "tree") + +DEFAULT_MODEL = "cohere/north-mini-code:free" +TOOL_USE_MODEL = "openai/gpt-4.1-mini" +FREE_TOOL_USE_MODEL = "inclusionai/ling-3.0-flash-sante:free" +DEFAULT_BUDGET_USD = Decimal("1.00") + +#: The canonical Goose runtime bindings (``participant-release.example.json``). +GOOSE_GATEWAY_BINDINGS: tuple[dict[str, Any], ...] = ( + {"field": "inference_gateway_host", "transport": "env", "key": "OPENAI_HOST"}, + {"field": "inference_gateway_base_path", "transport": "env", "key": "OPENAI_BASE_PATH"}, + {"field": "inference_gateway_credential", "transport": "env", "key": "OPENAI_API_KEY"}, + { + "field": "provider_kind", + "transport": "env", + "key": "GOOSE_PROVIDER", + "value_map": {"openai_compatible": "openai"}, + }, + {"field": "state_dir", "transport": "env", "key": "GOOSE_PATH_ROOT"}, +) + + +class SeedError(RuntimeError): + """A precondition of the seed failed; nothing further is written.""" + + +def require_local_dev(environ: Mapping[str, str]) -> None: + """Refuse anything but an explicit, local development database.""" + if environ.get("CODE4ME_DEV_SEED") != "1" and environ.get("TEST_MODE", "").lower() != "true": + raise SeedError("set CODE4ME_DEV_SEED=1 (or TEST_MODE=true) to seed a development database") + if environ.get("DB_HOST") not in (None, "", "localhost", "127.0.0.1", "::1"): + raise SeedError("refusing to seed a database that is not on this machine") + + +def profile_bindings(framework: str, *, gateway_tools: bool) -> list[dict[str, Any]]: + """The release's profile-field bindings (plus Goose's gateway runtime bindings).""" + if framework == "goose": + bindings: list[dict[str, Any]] = [ + {"field": "model", "transport": "env", "key": "GOOSE_MODEL"}, + {"field": "max_steps", "transport": "env", "key": "GOOSE_MAX_TURNS"}, + { + "field": "approval_policy", + "transport": "env", + "key": "GOOSE_MODE", + "value_map": {"suggestion_only": "chat", "per_step": "approve", "auto": "auto"}, + }, + *[dict(binding) for binding in GOOSE_GATEWAY_BINDINGS], + ] + if gateway_tools: + bindings.append( + {"field": "tools", "transport": "gateway", "key": "tool_allowlist", "format": "json"} + ) + return bindings + if gateway_tools: + raise SeedError("only a Goose arm can have its tools enforced at the research gateway") + return [ + {"field": "model", "transport": "env", "key": "CODEX_MODEL"}, + {"field": "max_steps", "transport": "env", "key": "CODEX_MAX_TURNS"}, + { + "field": "approval_policy", + "transport": "env", + "key": "INITIAL_AGENT_MODE", + "value_map": {"suggestion_only": "read-only", "per_step": "agent", "auto": "agent-full-access"}, + }, + ] + + +def build_release( + framework: str, + version: str, + *, + gateway_tools: bool, + tested_at: datetime, + platform: Optional[tuple[str, str]] = None, +) -> Any: + """A qualified BYOA release record for the checked, installed agent.""" + from research.study.agents.enums import DistributionMode + from research.study.agents.models import ( + AdapterRef, + AgentConfigBinding, + AgentReleaseV1, + ReleaseTests, + ) + from research.study.agents.registry import derive_qualification_status + + sys.path.insert(0, str(REPO)) + from scripts.dev.seed_research_study import detect_arch, detect_os # noqa: E402 + + os_name, arch = platform or (detect_os(), detect_arch()) + command, args, package = ("goose", ["acp"], "goose") if framework == "goose" else ("codex-acp", [], "codex") + bindings = profile_bindings(framework, gateway_tools=gateway_tools) + evidence = { + "framework": framework, "version": version, "command": command, "args": args, + "bindings": bindings, "tested_at": tested_at.isoformat(), + } + digest = "sha256:" + hashlib.sha256(json.dumps(evidence, sort_keys=True).encode()).hexdigest() + release_id = f"local-telemetry-{framework}-{version}" + ("-gateway-tools-v1" if gateway_tools else "") + release = AgentReleaseV1( + agent_id=framework, release_id=release_id, version=version, + source_manifest_digest=digest, distribution_mode=DistributionMode.BYOA_EXTERNAL, + agent_command=command, agent_command_args=args, agent_package=package, + byoa_config=[AgentConfigBinding(**binding) for binding in bindings], + # Base ACP normalisation only; the proxy accepts this id. + adapter=AdapterRef(adapter_id="generic-acp", version="1.0.0"), + # The self-check ran on this machine, so the release qualifies for it. + tests=[ReleaseTests(os=os_name, arch=arch, self_check="PASS", acp_initialize="PASS", ran_at=tested_at)], + ) + # Qualification is what the registry derives from the recorded tests. + status = derive_qualification_status(release.model_dump(mode="json")) + return release.model_copy(update={"qualification_status": status}) + + +@dataclass(frozen=True) +class StudyPlan: + framework: str + model: str + interaction: bool + gateway_tools: bool + budget_micro_usd: int + input_usd_per_million: Decimal + output_usd_per_million: Decimal + + @property + def kind(self) -> str: + if self.gateway_tools: + return "interaction-gateway-tools" + return "interaction" if self.interaction else "telemetry" + + def profile_fields(self) -> dict[str, Any]: + return { + "model": self.model, + "framework_version": self.framework, + "tools_json": json.dumps(list(GOOSE_TOOL_USE_SELECTION)) if self.gateway_tools else "[]", + "approval_policy": "per_step" if self.interaction else "suggestion_only", + "max_steps": 8 if self.interaction else 1, + } + + +def default_price(model: str) -> tuple[Decimal, Decimal]: + """A development price: free models cost nothing, others a placeholder.""" + if model.endswith(":free"): + return Decimal("0"), Decimal("0") + return Decimal("0.40"), Decimal("1.60") + + +def check_agent(command: Sequence[str], version_command: Sequence[str], environment: Mapping[str, str], + cwd: Optional[str] = None) -> str: + """The agent's version, after a successful ACP ``initialize`` handshake.""" + version = subprocess.run( + list(version_command), check=True, capture_output=True, text=True, timeout=30, + env=dict(environment), cwd=cwd, + ).stdout.strip() + if not version: + raise SeedError("the agent's version check printed nothing") + process = subprocess.Popen( + list(command), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + text=True, bufsize=1, env=dict(environment), cwd=cwd, + ) + lines: "queue.Queue[Optional[str]]" = queue.Queue() + + def read_lines() -> None: + for line in process.stdout: + lines.put(line) + lines.put(None) # end of stream: the agent exited or closed stdout + + reader = threading.Thread(target=read_lines, daemon=True) + try: + assert process.stdin is not None and process.stdout is not None + reader.start() + process.stdin.write(json.dumps({ + "jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": 1, "clientCapabilities": {}, + "clientInfo": {"name": "code4me-local-release-check", "version": "1"}}, + }) + "\n") + process.stdin.flush() + deadline = time.monotonic() + 30 + while True: + try: + line = lines.get(timeout=max(0.0, deadline - time.monotonic())) + except queue.Empty: + raise SeedError("ACP initialize did not complete within 30 s") from None + if line is None: + raise SeedError("the agent exited before answering ACP initialize") + try: + response = json.loads(line) + except json.JSONDecodeError as error: + raise SeedError("the agent wrote a non-JSON line on its ACP stdout") from error + if response.get("id") == 1: + if (response.get("result") or {}).get("protocolVersion") != 1: + raise SeedError("ACP initialize did not negotiate protocol version 1") + return version.rsplit(" ", 1)[-1] + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + + +def seed(session: Any, plan: StudyPlan, release: Any, *, now: datetime) -> dict[str, Any]: + """Write the release, priced connection, profile and active study (idempotent).""" + from sqlalchemy import select + + from database import db_schemas + from research.study import lifecycle as study_lifecycle + from research.study.agents import store as agent_store + from research.study.agents.distributions import validate_profile_configuration + from research.study.agents.enums import QualificationStatus + from research.study.protocol import store as study_store + + sys.path.insert(0, str(REPO)) + from scripts.dev.seed_research_study import ( # noqa: E402 - repository-root script + FIXED_SCHEDULE_END, + FIXED_SCHEDULE_START, + ensure_dev_owner, + ensure_dev_provider_connection, + ) + + owner = ensure_dev_owner( + session, f"telemetry-{plan.framework}-owner@example.com", + password=secrets.token_urlsafe(32), name=f"Local {plan.framework.title()} Telemetry Owner", + ) + connection = ensure_dev_provider_connection(session) + if not connection.is_active: + raise SeedError("the development provider connection is inactive") + models = json.loads(connection.models_json or "[]") + if plan.model not in models: + connection.models_json = json.dumps([*models, plan.model]) + session.add(connection) + if session.get(db_schemas.ProviderModelPrice, (connection.connection_id, plan.model)) is None: + session.add(db_schemas.ProviderModelPrice( + connection_id=connection.connection_id, model=plan.model, + input_usd_per_million=plan.input_usd_per_million, + output_usd_per_million=plan.output_usd_per_million, + updated_by="seed_local_byoa_telemetry_study", + )) + session.commit() + + stored = agent_store.get_release(session, release.release_id) + if stored is None: + stored = agent_store.upsert_release(session, release) + if stored.status != QualificationStatus.QUALIFIED.value: + raise SeedError(f"release {release.release_id} is not qualified: {stored.status}") + + profile_name = f"local-{plan.kind}-{plan.framework}" + profile = session.execute(select(db_schemas.AgentProfile).where( + db_schemas.AgentProfile.owner_user_id == owner.user_id, + db_schemas.AgentProfile.name == profile_name, + )).scalars().first() + if profile is None: + profile = db_schemas.AgentProfile( + profile_id=uuid.uuid4(), owner_user_id=owner.user_id, name=profile_name, + release_id=release.release_id, connection_id=connection.connection_id, + is_active=True, **plan.profile_fields(), + ) + # The same invariant the profile API and the study freeze apply. + validate_profile_configuration(profile, release) + session.add(profile) + session.commit() + session.refresh(profile) + + study_id = uuid.uuid5(uuid.NAMESPACE_URL, f"code4me2://local-{plan.kind}/{plan.framework}") + if study_store.get_study(session, study_id) is None: + study_store.create_study( + session, study_id=study_id, + name=f"Local {plan.framework.title()} {plan.kind.replace('-', ' ').title()} Verification", + description="Development-only agent telemetry verification; not a research study.", + created_by=owner.user_id, starts_at=FIXED_SCHEDULE_START, ends_at=FIXED_SCHEDULE_END, + is_research=True, + research_config_json={ + "telemetry_policy": {"metadata_only": True}, + "session_policy": {"idle_timeout_seconds": 900, "resume_grace_seconds": 300, + "heartbeat_seconds": 30}, + }, + join_code=study_lifecycle.allocate_join_code(session), + profile_ids=[profile.profile_id], + now=now, + inference_budget_default_micro_usd=plan.budget_micro_usd, + ) + study_store.set_study_active(session, study_id, True) + study = study_store.get_study(session, study_id) + stored_budget = session.get(db_schemas.Study, study_id).inference_budget_default_micro_usd + summary = { + "study_id": str(study_id), "join_code": study.join_code, "release_id": profile.release_id, + "profile_id": str(profile.profile_id), "model": profile.model, + "budget_usd": f"{Decimal(stored_budget or 0) / Decimal(1_000_000):.2f}", + } + # A re-run keeps what exists; say so instead of printing values it did not apply. + summary["notes"] = [ + f"existing {what} kept: {stored!r} (requested {wanted!r})" + for what, stored, wanted in ( + ("profile release", profile.release_id, release.release_id), + ("profile model", profile.model, plan.model), + ("study default budget (micro-USD)", stored_budget, plan.budget_micro_usd), + ) + if stored != wanted + ] + return summary + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("framework", choices=("goose", "codex")) + parser.add_argument("--interaction-test", action="store_true", + help="multi-step study with per-step approval") + tools = parser.add_mutually_exclusive_group() + tools.add_argument("--tool-use-test", action="store_true", + help=f"Goose: gateway-enforced tools {list(GOOSE_TOOL_USE_SELECTION)} on {TOOL_USE_MODEL}") + tools.add_argument("--free-tool-use-test", action="store_true", + help=f"as --tool-use-test on the free {FREE_TOOL_USE_MODEL}") + parser.add_argument("--model", default=None, help="override the model") + parser.add_argument("--budget-usd", type=Decimal, default=DEFAULT_BUDGET_USD, + help="default participant budget (Goose arms are metered)") + parser.add_argument("--goose", default=None, help="Goose executable (default: goose on PATH)") + parser.add_argument("--codex-acp", default=None, + help="the codex-acp executable the release launches (default: codex-acp on PATH)") + return parser + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + gateway_tools = args.tool_use_test or args.free_tool_use_test + if gateway_tools and args.framework != "goose": + parser.error("--tool-use-test/--free-tool-use-test need the goose framework") + model = args.model or ( + FREE_TOOL_USE_MODEL if args.free_tool_use_test else TOOL_USE_MODEL if args.tool_use_test else DEFAULT_MODEL + ) + price_in, price_out = default_price(model) + plan = StudyPlan( + framework=args.framework, model=model, interaction=args.interaction_test or gateway_tools, + gateway_tools=gateway_tools, budget_micro_usd=int(args.budget_usd * 1_000_000), + input_usd_per_million=price_in, output_usd_per_million=price_out, + ) + + from dotenv import load_dotenv + + load_dotenv() + require_local_dev(os.environ) + + environment = os.environ.copy() + if args.framework == "goose": + goose = args.goose or shutil.which("goose") + if not goose: + raise SeedError("no goose executable on PATH; pass --goose") + version = check_agent([goose, "acp"], [goose, "--version"], environment) + else: + # The release launches `codex-acp`; check that very executable. + codex_acp = args.codex_acp or shutil.which("codex-acp") + if not codex_acp: + raise SeedError("no codex-acp executable on PATH; install one (or link the vendored " + "adapter) or pass --codex-acp") + environment.update(CODEX_MODEL=model) + version = check_agent([codex_acp], [codex_acp, "--version"], environment) + print(f"{args.framework} {version}: version check and ACP initialize passed") + + from App import App + + session = App.get_instance().get_db_session() + try: + now = datetime.now(timezone.utc) + summary = seed(session, plan, build_release(args.framework, version, gateway_tools=gateway_tools, tested_at=now), now=now) + finally: + session.close() + for note in summary.pop("notes"): + print(f"note: {note}") + for key, value in summary.items(): + print(f"{key}={value}") + print("No participant was enrolled or consented; join with the code above.") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except SeedError as error: + print(f"seed refused: {error}", file=sys.stderr) + sys.exit(2) diff --git a/scripts/dev/seed_local_dev.sh b/scripts/dev/seed_local_dev.sh new file mode 100755 index 0000000..2249d99 --- /dev/null +++ b/scripts/dev/seed_local_dev.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# +# One-command local-dev seed for a fresh database. +# +# It stages the *built* runtime manifest (and any archives that exist next to +# it) into the running backend container, then runs the canonical seeder +# (scripts/dev/seed_research_study.py --fresh-db). The seeder imports the +# release with producer test results, +# pins default-code4me2-agent to it, turns Goose/Codex into BYOA identities, and +# publishes one study revision with a working session policy. +# +# No digest or size is ever typed by hand: when an archive is present its real +# sha256/size are computed inside the container and checked against the manifest. +# +# Usage (from code4me2-server/): +# MANIFEST=dist/release-1.2.3/native-macos-arm64.json scripts/dev/seed_local_dev.sh +# MANIFEST=dist/release-1.2.3/native-macos-arm64.json scripts/dev/seed_local_dev.sh --builtin-study-name "My Study" +# +# Environment overrides: +# CONTAINER backend container name (default: backend) +# MANIFEST producer manifest JSON with its ZIP next to it (required; the plugin ships none) +# STAGING in-container staging directory (default: /tmp/code4me-runtime-staging) + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +CONTAINER="${CONTAINER:-backend}" +MANIFEST="${MANIFEST:-}" +STAGING="${STAGING:-/tmp/code4me-runtime-staging}" + +if [[ ! -f "$MANIFEST" ]]; then + echo "manifest not found: $MANIFEST" >&2 + echo "set MANIFEST=/path/to/manifest.json (or build the runtime first)" >&2 + exit 1 +fi + +# The producer owns test results; never invent them at seed time. An untested +# manifest is rejected before anything touches the database. +if ! python3 - "$MANIFEST" <<'PY' +import json +import sys + +document = json.load(open(sys.argv[1], encoding="utf-8")) +artifacts = document.get("artifacts") or [] + + +def passed(artifact): + tests = artifact.get("tests") + return ( + isinstance(tests, dict) + and tests.get("self_check") == "PASS" + and tests.get("acp_initialize") == "PASS" + and bool(tests.get("ran_at")) + ) + + +raise SystemExit(0 if artifacts and all(passed(a) for a in artifacts) else 1) +PY +then + cat >&2 < --platform macos-arm64 \\ + --server-commit "\$(git rev-parse HEAD)" --output dist/release- + MANIFEST=dist/release-/native-macos-arm64.json scripts/dev/seed_local_dev.sh + +Omit --skip-build to build, test and archive the agent from source. +EOF + exit 1 +fi + +MANIFEST_DIR="$(cd "$(dirname "$MANIFEST")" && pwd)" +MANIFEST_NAME="$(basename "$MANIFEST")" + +echo "==> staging the runtime archives into container '$CONTAINER'" +docker exec "$CONTAINER" bash -lc "rm -rf '$STAGING' && mkdir -p '$STAGING/code4me-runtime'" +# The source manifest carries the pinned runtime version (and the archive +# basenames); keep its test results and digest identity unchanged. +docker cp "$MANIFEST" "$CONTAINER:$STAGING/code4me-runtime/$MANIFEST_NAME" >/dev/null + +shopt -s nullglob +ARCHIVE_COUNT=0 +for archive in "$MANIFEST_DIR"/code4me-agent-*.zip; do + base="$(basename "$archive")" + echo " + $base" + docker cp "$archive" "$CONTAINER:$STAGING/code4me-runtime/$base" + ARCHIVE_COUNT=$((ARCHIVE_COUNT + 1)) +done +shopt -u nullglob +if [[ "$ARCHIVE_COUNT" -eq 0 ]]; then + echo "no code4me-agent-*.zip archives found next to $MANIFEST" >&2 + exit 1 +fi + +# Use the producer's tested manifest unchanged; do not manufacture test results. +echo "==> seeding database (verified import -> pin -> publish)" +docker exec -i "$CONTAINER" bash -lc "source activate myenv && cd /app && CODE4ME_DEV_SEED=1 python scripts/dev/seed_research_study.py --manifest '$STAGING/code4me-runtime/$MANIFEST_NAME' --archives-dir '$STAGING/code4me-runtime' \"\$@\"" -- "$@" diff --git a/scripts/dev/seed_research_study.py b/scripts/dev/seed_research_study.py new file mode 100644 index 0000000..551b428 --- /dev/null +++ b/scripts/dev/seed_research_study.py @@ -0,0 +1,921 @@ +#!/usr/bin/env python3 +"""Seed a synthetic research study for local onboarding. + +This is the operator tool the participant runbook uses to mint a study-owned +*join code*. It performs no domain logic of its own: every state transition delegates +to the real research platform services and stores: + +* ``research.study.protocol.store`` (``create_study`` / ``get_study`` / + ``set_study_active`` / ``get_study_by_join_code``); +* ``research.study.lifecycle`` (``allocate_join_code`` for the study-owned join + code, ``open_study_enrollment`` for the atomic web-consent enrollment); +* ``research.study.agents.store`` (``upsert_release``) with + ``research.study.agents.models.AgentReleaseV1`` (qualification is derived from + the release's producer tests, not supplied by the seeder); +* ``database.crud`` for the login account. + +Everything is digest-pinned and deterministic so the script is safe to re-run: +the same arguments resolve the same account, release, study and enrollment +instead of piling up duplicates. Nothing secret is printed except +a password the operator explicitly supplied with ``--create-account``. + +Usage (from ``code4me2-server/``, development/test only):: + + CODE4ME_DEV_SEED=1 PYTHONPATH=src python scripts/dev/seed_research_study.py \\ + --account-email participant@example.com \\ + --account-password 'Password123' --create-account + +``TEST_MODE=true`` is accepted in place of ``CODE4ME_DEV_SEED=1``. Without one of +those guards the script exits before touching a database, so it can never seed a +production deployment by accident. + +The printed ``join_code`` is the study-owned code the participant redeems +through web consent; enrollment itself happens there, never in the plugin. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import sys +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional, Sequence +from uuid import UUID + +from research.study import lifecycle as study_lifecycle +from research.study.agents.enums import ( + DistributionMode, + QualificationStatus, +) +from research.study.agents.manifest_import import build_manifest_release +from research.study.agents.registry import AgentRegistry +from research.study.protocol import store as protocol_store + +if TYPE_CHECKING: + from research.study.agents.models import AgentReleaseV1 + +# The fixed window is deliberately deterministic (not "now + N days") so the +# seeded study schedule is identical on every run. ``end_at`` stays far enough +# in the future for the synthetic study to remain open. +FIXED_SCHEDULE_START = datetime(2024, 1, 1, tzinfo=timezone.utc) +FIXED_SCHEDULE_END = datetime(2099, 1, 1, tzinfo=timezone.utc) + +DEFAULT_STUDY_NAME = "Synthetic Onboarding Study" +DEFAULT_AGENT_ID = "code4me-synthetic-agent" +DEFAULT_RELEASE_VERSION = "1.0.0" +DEFAULT_ARTIFACT_SIZE = 1024 + +#: The fresh-DB seeder pins the *built-in* Code4Me runtime distribution and +#: leaves the participant-installed runtimes as BYOA identities. +DEFAULT_BUILTIN_STUDY_NAME = "Code4Me2 Built-in Agent Study" +BUILTIN_DISTRIBUTION_NAME = "default-code4me2-agent" +#: Development-only account created by ``--fresh-db`` seeding. Guarded by the +#: CODE4ME_DEV_SEED environment variable in ``scripts/dev/seed_local_dev.sh`` +#: and by the ``--fresh-db`` flag; never a production credential. +DEFAULT_DEV_OWNER_EMAIL = "research-owner@local.dev" +DEFAULT_DEV_PASSWORD = "Code4me-dev1" +#: ``(profile name, discovery package, command)`` for the BYOA profiles. +BYOA_PROFILE_IDENTITIES = ( + ("default-goose", "goose", "goose"), + ("default-codex", "codex", "codex"), +) +#: Development-only provider connection created by ``--fresh-db`` seeding. +#: The research UI only lists admin-created connections, so without this a fresh +#: database can never author a profile from the UI. Only the secret's env-var +#: name is stored; the value is resolved from the environment at inference time. +DEFAULT_DEV_CONNECTION_LABEL = "local-dev-openrouter" +DEFAULT_DEV_CONNECTION_BASE_URL = "https://openrouter.ai/api/v1" +DEFAULT_DEV_CONNECTION_SECRET_REF = "OPENROUTER_API_KEY" +DEFAULT_DEV_CONNECTION_MODELS: tuple[str, ...] = ( + "cohere/north-mini-code:free", + "openai/gpt-4o-mini", +) + + +class SeedError(RuntimeError): + """A typed, operator-facing failure (no partial policy is silently kept).""" + + +@dataclass(frozen=True) +class SeedRequest: + """The fully resolved inputs of one seed run.""" + + account_email: str + account_password: Optional[str] + account_name: str + create_account: bool + config_id: Optional[int] + study_name: str + agent_id: str + release_id: str + release_version: str + artifact_digest: str + artifact_path: str + artifact_size: int + os_name: str + arch: str + actor: str + require_capabilities: bool = False + # Distribution mode: PACKAGED (default, digest-pinned artifact) or + # BYOA_EXTERNAL (participant-installed Goose/Codex/...; command/package + # identity instead of an artifact digest). + distribution_mode: str = DistributionMode.PACKAGED.value + agent_command: Optional[str] = None + agent_command_args: tuple[str, ...] = () + agent_package: Optional[str] = None + # Optional link to an existing, server-side AgentProfile (provider/model/ + # tools/policy catalogue). Only the opaque id is pinned into the protocol; + # the profile's provider/base_url/api_key_ref never enter the document. + agent_profile_id: Optional[UUID] = None + + @property + def is_byoa(self) -> bool: + return self.distribution_mode.strip().upper() == DistributionMode.BYOA_EXTERNAL.value + + @property + def study_id(self) -> UUID: + """Deterministic study identity derived from the study name.""" + return uuid.uuid5( + uuid.NAMESPACE_URL, f"code4me2://research/study/{self.study_name}" + ) + + @property + def distribution_id(self) -> UUID: + """Deterministic distribution (AgentProfile) identity. + + The distribution is the single thing a researcher selects; the seed mints + a deterministic one per (study, release, mode) so re-runs are idempotent. + """ + return uuid.uuid5( + uuid.NAMESPACE_URL, + "code4me2://research/distribution/" + f"{self.study_name}/{self.release_id}/{self.distribution_mode}", + ) + + +@dataclass(frozen=True) +class FreshDbRequest: + """The resolved inputs of a "fresh DB immediately usable" seed run. + + It imports the built runtime manifest (never a synthetic digest), approves + the release for the built platforms so it is QUALIFIED, pins the built-in + distribution to that release, marks the participant-installed profiles as + BYOA, and creates one live study with a working session policy. + """ + + manifest: dict[str, Any] + #: Directory holding the manifest's declared archives (basenames). The + #: ``code4me-runtime/`` resource subdirectory is also searched. + archives_dir: Optional[str] = None + study_name: str = DEFAULT_BUILTIN_STUDY_NAME + actor: str = "seed-script" + require_capabilities: bool = False + # Development owner account that owns the seeded profile and study. It is + # created enabled-for-research when missing so a fresh database is usable + # without any per-participant SQL or seed migration. + owner_email: str = DEFAULT_DEV_OWNER_EMAIL + owner_password: str = DEFAULT_DEV_PASSWORD + owner_name: str = "Research Owner" + # Development provider connection the built-in profile is wired to. + connection_label: str = DEFAULT_DEV_CONNECTION_LABEL + connection_base_url: str = DEFAULT_DEV_CONNECTION_BASE_URL + connection_secret_ref: str = DEFAULT_DEV_CONNECTION_SECRET_REF + connection_models: tuple[str, ...] = DEFAULT_DEV_CONNECTION_MODELS + profile_model: str = DEFAULT_DEV_CONNECTION_MODELS[0] + + @property + def study_id(self) -> UUID: + """Deterministic study identity derived from the study name.""" + return uuid.uuid5( + uuid.NAMESPACE_URL, f"code4me2://research/study/{self.study_name}" + ) + + +@dataclass(frozen=True) +class FreshDbSummary: + """The result of a fresh-DB seed run (non-secret, copy-pasteable).""" + + agent_id: str + release_id: str + release_version: str + qualification: str + distribution_id: str + distribution_verified: bool + supported_platforms: list[dict[str, str]] + study_id: str + join_code: str + session_policy: dict[str, Any] + verified_artifacts: list[dict[str, Any]] + provider_connection_label: str = "" + provider_connection_models: tuple[str, ...] = () + profile_model: str = "" + + +# --------------------------------------------------------------------------- +# Pure planning helpers +# --------------------------------------------------------------------------- + + +def detect_os() -> str: + """Return the Gradle/plugin platform name for the current host.""" + if sys.platform.startswith("win"): + return "windows" + if sys.platform == "darwin": + return "macos" + return "linux" + + +def detect_arch() -> str: + """Return the Gradle/plugin architecture name for the current host.""" + machine = platform.machine().lower() + if machine in ("aarch64", "arm64"): + return "aarch64" + return "x64" + + +def build_research_config( + request: SeedRequest, + *, + profile_id: UUID, +) -> dict[str, Any]: + """The deterministic ``research_config_json`` this script stores on the study. + + Mirrors the ``POST /api/research/studies`` shape: a telemetry policy, a + fully populated session policy (so the sessions endpoints can open a + session instead of returning ``POLICY_MISSING``), and the selected + agent profile ids. + """ + return { + "telemetry_policy": {"metadata_only": True}, + "session_policy": { + "idle_timeout_seconds": 900, + "resume_grace_seconds": 300, + "heartbeat_seconds": 30, + }, + "profile_ids": [str(profile_id)], + } + + +# --------------------------------------------------------------------------- +# Account resolution +# --------------------------------------------------------------------------- + + +def _registry_from_db(session: Any) -> AgentRegistry: + from research.study.agents import store as agents_store + + registry = AgentRegistry() + for row in agents_store.list_releases(session): + registry.register_release(agents_store.row_to_release(row)) + return registry + + +def ensure_study( + session: Any, + request: SeedRequest, + research_config: dict[str, Any], + *, + owner_user_id: Any = None, + profile_id: UUID, +) -> Any: + """Create the deterministic study identity if it does not exist yet. + + The study owns its join code and selects the seeded agent profile, exactly + like ``POST /api/research/studies``. Safe to re-run: an existing study is + returned untouched. + """ + row = protocol_store.get_study(session, request.study_id) + if row is None: + row = protocol_store.create_study( + session, + study_id=request.study_id, + name=request.study_name, + description="Synthetic study seeded for local onboarding.", + created_by=owner_user_id, + starts_at=FIXED_SCHEDULE_START, + ends_at=FIXED_SCHEDULE_END, + is_research=True, + research_config_json=research_config, + join_code=study_lifecycle.allocate_join_code(session), + profile_ids=[profile_id], + ) + return row + if getattr(row, "research_status", None) != "STUDY_STOPPED": + return row + # A stopped study is terminal (``set_study_active`` refuses it), but a + # re-seed must still yield a live study. Reuse an earlier replacement when it + # is still open, otherwise create one, so re-running stays idempotent. + replacement_name = f"{request.study_name} (reseeded)" + existing = _study_by_name(session, replacement_name) + if existing is not None and getattr(existing, "research_status", None) != "STUDY_STOPPED": + view = protocol_store.get_study(session, existing.study_id) + if view is not None: + return view + return protocol_store.create_study( + session, + study_id=uuid.uuid4(), + name=replacement_name, + description="Synthetic study seeded for local onboarding (replaces a stopped study).", + created_by=owner_user_id, + starts_at=FIXED_SCHEDULE_START, + ends_at=FIXED_SCHEDULE_END, + is_research=True, + research_config_json=research_config, + join_code=study_lifecycle.allocate_join_code(session), + profile_ids=[profile_id], + ) + + +def _study_by_name(session: Any, name: str) -> Any: + """Return the study row with ``name``, or ``None`` (seed helper).""" + from sqlalchemy import select + + from database.db_schemas import Study as StudyRow + + return session.execute(select(StudyRow).where(StudyRow.name == name)).scalars().first() + + +def activate_study(session: Any, study_id: UUID) -> None: + """Mark the seeded research study live, as the study routes do. + + Activation reserves the owner's single live-study slot (``is_active``); + without it the assignment path refuses the study as not live. Idempotent, + so re-running the seed is safe. + """ + row = protocol_store.set_study_active(session, study_id, True) + if row is None: + raise SeedError(f"seeded study {study_id} has no study row") + + +def resolve_declared_archives( + manifest: dict[str, Any], archives_dir: Optional[str] +) -> dict[str, Any]: + """Map every archive the manifest declares to its on-disk bytes. + + ``archive`` fields are basenames; the seeder resolves each against the + supplied directory (and a ``code4me-runtime/`` subdirectory, the layout of a + locally built plugin recipe). A declared archive that cannot be + found is left out: the planner then rejects the import, exactly like the + HTTP route does. + """ + from pathlib import Path + + roots: list[Path] = [] + if archives_dir: + roots.append(Path(archives_dir)) + roots.append(Path(archives_dir) / "code4me-runtime") + found: dict[str, Path] = {} + for raw in manifest.get("artifacts") or []: + if not isinstance(raw, dict): + continue + name = Path(str(raw.get("archive") or "")).name + if not name or name in found: + continue + for root in roots: + candidate = root / name + if candidate.is_file(): + found[name] = candidate + break + return found + + +def import_manifest_release( + session: Any, request: FreshDbRequest +) -> tuple[AgentReleaseV1, list[dict[str, Any]]]: + """Register the release the build manifest + archives describe (idempotently). + + The manifest is mapped through the same pure planner the import endpoint + uses, so the seeder and CI resolve the exact same release identity. Every + declared archive must be present and match its digest and size; a mismatch + raises rather than silently trusting the manifest. + """ + from research.study.agents import store as agents_store + + archives = resolve_declared_archives(request.manifest, request.archives_dir) + try: + plan = build_manifest_release(request.manifest, archives=archives) + except Exception as error: # noqa: BLE001 - re-raise as an operator-facing error + raise SeedError(f"build manifest could not be imported: {error}") from error + + desired = plan.release + registry = _registry_from_db(session) + existing = registry.get_release(desired.agent_id, desired.release_id) + if existing is None: + registration = registry.register_release(desired) + if not registration.accepted: + raise SeedError( + "release could not be registered: " + f"{registration.issue.message if registration.issue else 'unknown'}" + ) + agents_store.upsert_release(session, desired) + elif existing.digest_identity != desired.digest_identity: + raise SeedError( + f"release {desired.release_id!r} already exists with a different digest " + "identity; the manifest is immutable -- remove the stale release first." + ) + + row = agents_store.get_release(session, desired.release_id) + if row is None: # pragma: no cover - defensive + raise SeedError(f"release {desired.release_id!r} vanished after registration") + return agents_store.row_to_release(row), plan.verified_artifacts + + +def ensure_dev_provider_connection( + session: Any, + *, + label: str = DEFAULT_DEV_CONNECTION_LABEL, + base_url: str = DEFAULT_DEV_CONNECTION_BASE_URL, + secret_ref: str = DEFAULT_DEV_CONNECTION_SECRET_REF, + models: tuple[str, ...] = DEFAULT_DEV_CONNECTION_MODELS, +) -> Any: + """Return the development provider connection, creating it when missing. + + A fresh database has none, and the research UI only lists admin-created + connections, so a seeded profile could never be authored from the UI. The + row stores the secret's environment-variable name only; the value is resolved + from the environment at inference time and is never persisted here. + """ + from database import crud + + if not models: + raise SeedError("a development provider connection needs at least one model") + existing = crud.get_provider_connection_by_label(session, label) + if existing is not None: + return existing + return crud.create_provider_connection( + session, + label=label, + base_url=base_url, + secret_ref=secret_ref, + models_json=json.dumps(list(models)), + is_active=True, + ) + + +def ensure_dev_owner( + session: Any, + email: str, + *, + password: str = DEFAULT_DEV_PASSWORD, + name: str = "Research Owner", +) -> Any: + """Return the development owner account, creating/enabling it if needed. + + The account is administrator-independent: it is created *enabled for + research* (``can_research``) so a fresh database can author its first study + without a per-study role grant. Existing accounts are left alone except for + the enable flag. + """ + from database import crud + + user = crud.get_user_by_email(session, email) + if user is None: + configs = crud.get_all_configs(session) + if not configs: + raise SeedError( + "no configuration row exists; run init.sql/migrations before " + "seeding a development owner account." + ) + import Queries + + user = crud.create_user( + session, + Queries.CreateUser( + email=email, + name=name, + password=password, + config_id=int(configs[0].config_id), + ), + ) + if not user.can_research or not user.verified: + user.can_research = True + user.verified = True + session.add(user) + session.commit() + session.refresh(user) + return user + + +def pin_builtin_distribution( + session: Any, + release: AgentReleaseV1, + *, + name: str = BUILTIN_DISTRIBUTION_NAME, + owner_user_id: Any = None, + connection_id: Any = None, + model: str = DEFAULT_DEV_CONNECTION_MODELS[0], +) -> Any: + """Pin the built-in Code4Me profile to ``release`` (idempotently). + + Profiles are now researcher-owned and are never seeded by a migration, so a + fresh database has no built-in profile. This creates one owned by the + development owner account when it is missing; a profile that already exists + only gets its release pin updated (and its provider wiring repaired when it + still carries the old placeholder model/connection). + """ + from sqlalchemy import select + + from database import db_schemas + + row = ( + session.execute( + select(db_schemas.AgentProfile).where( + db_schemas.AgentProfile.owner_user_id == owner_user_id, + db_schemas.AgentProfile.name == name, + ) + ) + .scalars() + .first() + ) + if row is None: + if owner_user_id is None: + raise SeedError( + f"cannot create built-in agent profile {name!r} without an owner " + "account (fresh-db seeding creates one automatically)." + ) + row = db_schemas.AgentProfile( + profile_id=uuid.uuid4(), + owner_user_id=owner_user_id, + name=name, + model=model, + framework_version="code4me2-agent", + tools_json="[]", + # Verified by the synthetic permission/edit cases the dev seed records. + approval_policy="suggestion_only", + max_steps=1, + is_active=True, + release_id=release.release_id, + connection_id=connection_id, + ) + session.add(row) + session.commit() + session.refresh(row) + return row + row.release_id = release.release_id + # Repair a profile seeded before connections existed (placeholder model and + # no connection), so an existing dev database becomes usable on a re-run. + if row.connection_id is None and connection_id is not None: + row.connection_id = connection_id + if (row.model or "").strip() in ("", "seed-model") and model: + row.model = model + session.add(row) + session.commit() + session.refresh(row) + return row + + +def seed_fresh_database(session: Any, request: FreshDbRequest) -> FreshDbSummary: + """Make a fresh database immediately usable for research. + + Registers the shipped runtime release from its producer manifest (the + manifest's passing platform tests make it usable), + pins ``default-code4me2-agent`` to it (Goose/Codex become BYOA), + and creates one live study whose ``session_policy`` is fully populated + so the sessions endpoints can open a session instead of returning + ``POLICY_MISSING``. Safe to re-run: every step resolves the existing row. + """ + from research.study.agents.distributions import ( + distribution_supported_platforms, + ) + + release, verified = import_manifest_release(session, request) + + owner = ensure_dev_owner(session, request.owner_email) + connection = ensure_dev_provider_connection( + session, + label=request.connection_label, + base_url=request.connection_base_url, + secret_ref=request.connection_secret_ref, + models=tuple(request.connection_models), + ) + profile = pin_builtin_distribution( + session, + release, + owner_user_id=owner.user_id, + connection_id=connection.connection_id, + model=request.profile_model or request.connection_models[0], + ) + + # Re-read the imported test-derived qualification. + from research.study.agents import store as agents_store + + release_row = agents_store.get_release(session, release.release_id) + if release_row is None: # pragma: no cover - defensive + raise SeedError(f"release {release.release_id!r} vanished") + release = agents_store.row_to_release(release_row) + + seed_request = SeedRequest( + account_email="", + account_password=None, + account_name="", + create_account=False, + config_id=None, + study_name=request.study_name, + agent_id=release.agent_id, + release_id=release.release_id, + release_version=release.version, + artifact_digest="", + artifact_path="", + artifact_size=0, + os_name="", + arch="", + actor=request.actor, + require_capabilities=request.require_capabilities, + ) + research_config = build_research_config(seed_request, profile_id=profile.profile_id) + study = ensure_study( + session, + seed_request, + research_config, + owner_user_id=profile.owner_user_id, + profile_id=profile.profile_id, + ) + # Also covers the idempotent path where the study already existed. + activate_study(session, study.study_id) + + live = protocol_store.get_study(session, study.study_id) + join_code = live.join_code if live is not None else None + return FreshDbSummary( + agent_id=release.agent_id, + release_id=release.release_id, + release_version=release.version, + qualification=release.qualification_status.value, + distribution_id=str(profile.profile_id), + distribution_verified=release.qualification_status == QualificationStatus.QUALIFIED, + supported_platforms=distribution_supported_platforms(release), + study_id=str(study.study_id), + join_code=str(join_code or ""), + session_policy=dict(research_config["session_policy"]), + verified_artifacts=verified, + provider_connection_label=connection.label, + provider_connection_models=tuple(json.loads(connection.models_json or "[]")), + profile_model=profile.model, + ) + + +def format_fresh_db_summary(summary: FreshDbSummary) -> str: + """Render the fresh-DB seed result (no secrets).""" + lines = [ + "Research database seeded from the built runtime manifest.", + f" agent_id: {summary.agent_id}", + f" release_id: {summary.release_id}", + f" release_version: {summary.release_version}", + f" qualification: {summary.qualification}", + f" distribution_id: {summary.distribution_id}", + f" distribution_verified: {summary.distribution_verified}", + f" supported_platforms: {summary.supported_platforms}", + f" study_id: {summary.study_id}", + f" join_code: {summary.join_code}", + f" session_policy: {summary.session_policy}", + f" provider_connection: {summary.provider_connection_label} {list(summary.provider_connection_models)}", + f" profile_model: {summary.profile_model}", + f" verified_artifacts: {summary.verified_artifacts}", + ] + return "\n".join(lines) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Seed research onboarding state. Imports the built runtime manifest " + "with its verified archive bytes, qualifies its release, pins the " + "built-in distribution and creates a live study. Safe to re-run; no " + "secrets are printed except a password supplied with " + "--account-password." + ) + ) + parser.add_argument( + "--account-email", + default=None, + help="participant login email (informational; the study is seeded without it)", + ) + parser.add_argument( + "--fresh-db", + action="store_true", + help=( + "accepted for backwards-compatible invocation; the manifest import is " + "the only seed path" + ), + ) + parser.add_argument( + "--manifest", + default=None, + help=( + "path to the producer runtime manifest JSON (use '-' to read it from " + "stdin); basename archives must be present for a successful import" + ), + ) + parser.add_argument( + "--archives-dir", + default=None, + help=( + "directory holding the manifest's declared archives (basenames); the " + "code4me-runtime/ resource subdirectory is also searched" + ), + ) + parser.add_argument( + "--account-password", + default=None, + help="participant password (required with --create-account)", + ) + parser.add_argument( + "--account-name", + default="Synthetic Participant", + help="display name used when creating the account", + ) + parser.add_argument( + "--create-account", + action="store_true", + help="create the login account if it does not already exist", + ) + parser.add_argument( + "--config-id", + type=int, + default=None, + help="completion config id for a new account (default: first available)", + ) + parser.add_argument("--study-name", default=DEFAULT_STUDY_NAME) + parser.add_argument("--agent-id", default=DEFAULT_AGENT_ID) + parser.add_argument("--release-id", default=None) + parser.add_argument("--release-version", default=DEFAULT_RELEASE_VERSION) + parser.add_argument( + "--artifact-digest", + default=None, + help="sha256:<64 hex> digest of the synthetic distribution artifact", + ) + parser.add_argument( + "--artifact-path", + default=None, + help="relative artifact path recorded on the release", + ) + parser.add_argument("--artifact-size", type=int, default=DEFAULT_ARTIFACT_SIZE) + parser.add_argument("--os", dest="os_name", default=None, help="artifact os") + parser.add_argument("--arch", default=None, help="artifact architecture") + parser.add_argument( + "--distribution-mode", + choices=[DistributionMode.PACKAGED.value, DistributionMode.BYOA_EXTERNAL.value], + default=DistributionMode.PACKAGED.value, + help=( + "PACKAGED (default) pins a digest-pinned artifact; BYOA_EXTERNAL pins " + "a participant-installed agent command/package" + ), + ) + parser.add_argument( + "--agent-command", + default=None, + help="BYOA only: participant-installed agent command (for example goose)", + ) + parser.add_argument( + "--agent-command-args", + default=None, + help="BYOA only: space-separated args appended after the agent command", + ) + parser.add_argument( + "--agent-package", + default=None, + help="BYOA only: logical package to discover when no command is given (goose/codex)", + ) + parser.add_argument( + "--profile-id", + "--agent-profile", + dest="profile_id", + type=UUID, + default=None, + help=( + "UUID of an existing AgentProfile to select for the study " + "(provider/model/tools/policy catalogue)" + ), + ) + parser.add_argument("--actor", default="seed-script", help="audit actor label") + parser.add_argument( + "--builtin-study-name", + default=DEFAULT_BUILTIN_STUDY_NAME, + help="study name used by --fresh-db", + ) + parser.add_argument( + "--connection-label", + default=DEFAULT_DEV_CONNECTION_LABEL, + help="--fresh-db: label of the development provider connection", + ) + parser.add_argument( + "--connection-base-url", + default=DEFAULT_DEV_CONNECTION_BASE_URL, + help="--fresh-db: upstream base URL for the development provider connection", + ) + parser.add_argument( + "--connection-secret-ref", + default=DEFAULT_DEV_CONNECTION_SECRET_REF, + help="--fresh-db: name of the deployment env var holding the provider key", + ) + parser.add_argument( + "--connection-model", + action="append", + default=[], + metavar="MODEL", + help=( + "--fresh-db: allowed model on the development provider connection " + "(repeatable; defaults to the built-in OpenRouter models)" + ), + ) + parser.add_argument( + "--profile-model", + default=None, + help="--fresh-db: model the built-in profile selects (default: first connection model)", + ) + parser.add_argument( + "--capability-requirements", + dest="require_capabilities", + action="store_true", + default=False, + help=( + "require host capability evidence (the compatibility gate). Off by " + "default: the plugin half of the gate is not implemented, so a study " + "that demands it cannot be joined." + ), + ) + return parser + + +def _load_manifest(source: str) -> dict[str, Any]: + """Load a manifest JSON document from a path or ``-`` (stdin).""" + if source == "-": + payload = json.load(sys.stdin) + else: + with open(source, "r", encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, dict): + raise SystemExit("manifest must be a JSON object") + return payload + + +def fresh_db_request_from_args(args: argparse.Namespace) -> FreshDbRequest: + """Build the fresh-DB seed request from parsed CLI arguments.""" + if not args.manifest: + raise SystemExit("--fresh-db requires --manifest PATH (or --manifest -).") + return FreshDbRequest( + manifest=_load_manifest(args.manifest), + archives_dir=args.archives_dir, + study_name=args.builtin_study_name, + actor=args.actor, + require_capabilities=args.require_capabilities, + connection_label=args.connection_label, + connection_base_url=args.connection_base_url, + connection_secret_ref=args.connection_secret_ref, + connection_models=tuple(args.connection_model) or DEFAULT_DEV_CONNECTION_MODELS, + profile_model=args.profile_model + or (tuple(args.connection_model) or DEFAULT_DEV_CONNECTION_MODELS)[0], + ) + + +def require_dev_guard() -> None: + """Fail fast unless this development-only seeder is explicitly enabled. + + The script mutates a real account/study/enrollment database, so it refuses + to run unless ``CODE4ME_DEV_SEED=1`` or ``TEST_MODE=true`` is set. This + makes it impossible to seed a production deployment by accident. + """ + if os.environ.get("CODE4ME_DEV_SEED") == "1": + return + if os.environ.get("TEST_MODE", "").strip().lower() in {"1", "true", "yes"}: + return + raise SystemExit( + "seed_research_study.py refuses to run: it is a development-only tool " + "that mutates a real database. Set CODE4ME_DEV_SEED=1 (or TEST_MODE=true) " + "to confirm you are seeding a local/dev/test database." + ) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + # Parse first so `--help` never requires the dev guard (or the app stack). + parser = build_parser() + args = parser.parse_args(argv) + require_dev_guard() + + from dotenv import load_dotenv + + load_dotenv() + + # Imported lazily so importing this module (for tests or --help) never + # pulls in the heavy application singleton / model stack. + from App import App + + app = App.get_instance() + session = app.get_db_session() + try: + # There is exactly one seed path: the built runtime manifest + its + # verified archive bytes. A synthetic release with an invented digest is + # no longer representable. + fresh_request = fresh_db_request_from_args(args) + summary = seed_fresh_database(session, fresh_request) + print(format_fresh_db_summary(summary)) + finally: + session.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/backup-postgres.sh b/scripts/ops/backup-postgres.sh new file mode 100755 index 0000000..5d0f4e9 --- /dev/null +++ b/scripts/ops/backup-postgres.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Nightly PostgreSQL backup for the Code4Me study database (production-readiness B-05). +# +# Runs inside the `backup` service of docker-compose.prod.yml (a postgres image, +# so `pg_dump` matches the server major) or on the host against a reachable DB. +# +# backup-postgres.sh one backup now +# backup-postgres.sh --loop one backup now, then every BACKUP_INTERVAL_SECONDS +# +# Environment (libpq variables plus): +# PGHOST PGPORT PGUSER PGPASSWORD PGDATABASE connection (compose sets them) +# BACKUP_DIR where dumps land (default /backups) +# BACKUP_RETENTION_DAYS prune dumps older than this (default 14; 0 disables) +# BACKUP_INTERVAL_SECONDS loop period (default 86400) +# BACKUP_OFFHOST_CMD optional command run with the dump path as $1 after a +# successful dump, e.g. `rclone copy "$1" remote:code4me` +# or `aws s3 cp "$1" s3://bucket/code4me/`; a failure is +# logged and does not delete the local copy. +# +# Output: /code4me--.dump (pg_dump custom +# format, compressed) plus a .sha256 sidecar. Restore with restore-postgres.sh. +set -euo pipefail + +BACKUP_DIR="${BACKUP_DIR:-/backups}" +BACKUP_RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-14}" +BACKUP_INTERVAL_SECONDS="${BACKUP_INTERVAL_SECONDS:-86400}" +PGDATABASE="${PGDATABASE:?PGDATABASE is required}" + +log() { printf '%s backup: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2; } + +run_once() { + mkdir -p "$BACKUP_DIR" + # A run killed mid-dump leaves a .partial behind; it is never a valid backup. + find "$BACKUP_DIR" -maxdepth 1 -type f -name '*.dump.partial' -mmin +120 -print -delete | sed 's/^/removed stale /' >&2 || true + local stamp file tmp offhost_ok=1 + stamp="$(date -u +%Y%m%dT%H%M%SZ)" + file="$BACKUP_DIR/code4me-${PGDATABASE}-${stamp}.dump" + tmp="$file.partial" + log "dumping $PGDATABASE from ${PGHOST:-localhost}:${PGPORT:-5432}" + if ! pg_dump --format=custom --compress=6 --no-owner --no-privileges --file="$tmp" "$PGDATABASE"; then + rm -f "$tmp" + log "pg_dump FAILED" + return 1 + fi + mv "$tmp" "$file" + ( cd "$BACKUP_DIR" && sha256sum "$(basename "$file")" > "$(basename "$file").sha256" ) + log "wrote $file ($(du -h "$file" | cut -f1))" + if [ -n "${BACKUP_OFFHOST_CMD:-}" ]; then + if bash -c "$BACKUP_OFFHOST_CMD" _ "$file"; then + log "off-host copy done" + else + offhost_ok=0 + log "off-host copy FAILED (local copies kept, retention skipped this run)" + fi + fi + # Never prune local dumps while the off-host copy is failing: the local + # disk is then the only copy there is. + if [ "$offhost_ok" -eq 1 ] && [ "$BACKUP_RETENTION_DAYS" -gt 0 ] 2>/dev/null; then + find "$BACKUP_DIR" -maxdepth 1 -type f \( -name 'code4me-*.dump' -o -name 'code4me-*.dump.sha256' \) \ + -mtime +"$BACKUP_RETENTION_DAYS" -print -delete | sed 's/^/pruned /' >&2 || true + fi +} + +if [ "${1:-}" = "--loop" ]; then + while true; do + run_once || true + sleep "$BACKUP_INTERVAL_SECONDS" + done +else + run_once +fi diff --git a/scripts/ops/restore-postgres.sh b/scripts/ops/restore-postgres.sh new file mode 100755 index 0000000..2406fa2 --- /dev/null +++ b/scripts/ops/restore-postgres.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Restore a backup-postgres.sh dump (production-readiness B-05). +# +# restore-postgres.sh --to-database [--create] drill into a scratch DB +# restore-postgres.sh --yes restore INTO $PGDATABASE (destructive) +# +# Environment: PGHOST PGPORT PGUSER PGPASSWORD PGDATABASE (libpq). Stop the +# backend and celery-worker before a destructive restore; the dump is applied +# with --clean --if-exists, so existing objects are replaced. +set -euo pipefail + +DUMP="${1:?usage: restore-postgres.sh (--to-database [--create] | --yes)}" +shift +TARGET="${PGDATABASE:-}" +CREATE=0 +CONFIRMED=0 +while [ $# -gt 0 ]; do + case "$1" in + --to-database) TARGET="$2"; shift 2 ;; + --create) CREATE=1; shift ;; + --yes) CONFIRMED=1; shift ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done +[ -f "$DUMP" ] || { echo "dump not found: $DUMP" >&2; exit 2; } +[ -n "$TARGET" ] || { echo "no target database (set PGDATABASE or --to-database)" >&2; exit 2; } +if [ -f "$DUMP.sha256" ]; then + ( cd "$(dirname "$DUMP")" && sha256sum -c "$(basename "$DUMP").sha256" ) || { echo "checksum mismatch" >&2; exit 1; } +fi +if [ "$TARGET" = "${PGDATABASE:-}" ] && [ "$CONFIRMED" -ne 1 ]; then + echo "refusing to overwrite $TARGET without --yes (use --to-database for a drill)" >&2 + exit 2 +fi +if [ "$CREATE" -eq 1 ]; then + createdb "$TARGET" || echo "createdb: database may already exist, continuing" >&2 +fi +echo "restoring $DUMP into $TARGET on ${PGHOST:-localhost}:${PGPORT:-5432}" >&2 +pg_restore --clean --if-exists --no-owner --no-privileges --dbname="$TARGET" "$DUMP" +echo "restore finished; verify with: psql -d $TARGET -c 'SELECT count(*) FROM public.research_event;'" >&2 diff --git a/session_id b/session_id new file mode 100644 index 0000000..e69de29 diff --git a/src/App.py b/src/App.py index 68aaf2c..1b79f9c 100644 --- a/src/App.py +++ b/src/App.py @@ -133,6 +133,7 @@ def __setup(self, config: Code4meV2Config) -> None: email_verification_token_expires_in_seconds=config.email_verification_token_expires_in_seconds, reset_password_token_expires_in_seconds=config.reset_password_token_expires_in_seconds, token_hook_activation_in_seconds=config.token_hook_activation_in_seconds, + password=config.redis_password, ) # if os env doesn't have variable CELERY_WORKER=TRUE run the following @@ -162,8 +163,9 @@ def __setup(self, config: Code4meV2Config) -> None: # Initialize completion models for AI functionality self.__completion_models = CompletionModels(config=config) - # Preload models if configured to do so for faster response times - if config.preload_models: + # Preload models if configured to do so for faster response times; switched-off + # classic models are never loaded + if config.preload_models and config.classic_models_enabled: logging.log(logging.INFO, "Preloading llm models...") db = self.get_db_session() @@ -317,10 +319,10 @@ def cleanup(self): This method performs cleanup in the reverse order of initialization to ensure dependencies are properly handled. """ - # Clean up Redis manager with a database session for final operations - self.__redis_manager.cleanup(db_session=self.__db_session_factory()) - - # Close Redis connection to stop background threads gracefully + # Only close: logins and sessions live in Redis and outlive this process. + # Flushing them here signed every user out on each restart, and with + # several workers sharing one Redis, whenever any one worker stopped. + # Sessions still end, and persist, through their expiry hooks. self.__redis_manager.close() # Clean up Celery broker connections diff --git a/src/Code4meV2Config.py b/src/Code4meV2Config.py index d5fee71..b808f80 100644 --- a/src/Code4meV2Config.py +++ b/src/Code4meV2Config.py @@ -236,13 +236,6 @@ class Code4meV2Config(BaseSettings): description="Frontend website port number", ) - react_app_google_client_id: str = Field( - alias="REACT_APP_GOOGLE_CLIENT_ID", - frozen=True, - min_length=1, - description="Google OAuth client ID for React application", - ) - # ----------------------- # Redis & Celery Configuration # ----------------------- @@ -254,6 +247,13 @@ class Code4meV2Config(BaseSettings): description="Redis server hostname or IP address", ) + redis_password: Optional[str] = Field( + alias="REDIS_PASSWORD", + default=None, + frozen=True, + description="Password for the session Redis (`requirepass`); empty disables AUTH", + ) + redis_port: int = Field( alias="REDIS_PORT", frozen=True, @@ -315,6 +315,26 @@ class Code4meV2Config(BaseSettings): description="Perform model warmup runs to optimize performance", ) + cookie_secure: bool = Field( + alias="COOKIE_SECURE", + default=False, + frozen=True, + description=( + "Set the Secure attribute on auth/session/project cookies; enable " + "behind HTTPS (production), leave off for plain-HTTP local stacks" + ), + ) + + classic_models_enabled: bool = Field( + alias="CLASSIC_MODELS_ENABLED", + default=True, + frozen=True, + description=( + "Serve the classic completion/chat model endpoints; when false they are " + "refused and no model is preloaded or loaded" + ), + ) + # ----------------------- # Thread Pool Configuration # ----------------------- diff --git a/src/Queries.py b/src/Queries.py index 2fde242..12ef213 100644 --- a/src/Queries.py +++ b/src/Queries.py @@ -11,11 +11,6 @@ from database.db_schemas import DEFAULT_USER_PREFERENCE -class Provider(Enum): - no_provider = "no_provider" - google = "google" - - class ContextChangeType(Enum): update = "update" insert = "insert" @@ -67,13 +62,6 @@ def validate_password(cls, v: SecretStr) -> SecretStr: return v -class CreateUserOauth(CreateUser): - token: str = Field(..., description="JWT token for authentication") - provider: Provider = Field( - ..., description="OAuth provider (Google, Microsoft, etc.)" - ) - - class UpdateUser(QueryBase): name: Optional[str] = Field( None, description="User's new name", min_length=3, max_length=50 @@ -141,11 +129,6 @@ def normalize_email(cls, v: str) -> str: return v.strip().lower() -class AuthenticateUserOAuth(QueryBase): - provider: Provider = Field(..., description="OAuth provider") - token: str = Field(..., description="OAuth token in JWT format") - - class CreateConfig(QueryBase): config_data: str = Field(..., description="Configuration data as JSON string") @@ -317,7 +300,9 @@ class CreateGeneration(QueryBase): generation_time: int = Field(..., description="Generation time (ms)", ge=0) shown_at: List[str] = Field(..., description="Timestamps when shown") was_accepted: bool = Field(..., description="Whether accepted by user") - confidence: float = Field(..., description="Confidence score") + confidence: Optional[float] = Field( + None, description="Confidence score; null when the model reports none" + ) logprobs: List[float] = Field(..., description="Token log probabilities") @@ -384,6 +369,41 @@ class ActivateProject(QueryBase): project_id: UUID = Field(..., description="Project id to activate") +class PrepareAcpGrant(QueryBase): + """Plugin request for a one-time launch grant for a local agent process. + + Both fields are *bound into* the grant, so the agent that redeems it can + only ever act on this project and inside this workspace directory. + """ + + project_id: UUID = Field( + ..., description="Activated project id for the ACP process" + ) + workspace: str = Field( + ..., description="Canonical project workspace", min_length=1, max_length=4096 + ) + launch_id: Optional[str] = Field( + default=None, + description="Stable identity for one managed runtime launch.", + min_length=1, + max_length=200, + ) + path_format: Optional[str] = Field( + default=None, + description="Client workspace syntax: posix or windows.", + pattern="^(posix|windows)$", + ) + managed_protocol_version: Optional[str] = Field( + default=None, + description="Managed-agent protocol requested by the plugin.", + pattern="^1$", + ) + + +class ExchangeAcpGrant(QueryBase): + grant: str = Field(..., description="One-time ACP launch grant", min_length=1) + + class CreateSession(QueryBase): user_id: UUID = Field(..., description="User ID") diff --git a/src/agents/__init__.py b/src/agents/__init__.py new file mode 100644 index 0000000..07e0bb2 --- /dev/null +++ b/src/agents/__init__.py @@ -0,0 +1,17 @@ +"""Server-side agent subsystem. + +Layout: + +* ``provider`` — resolves the generic OpenAI-compatible upstream from a profile +* ``normalize`` — wire-format normalization (Chat Completions / Responses API) +* ``tools`` — catalogue of selectable tool names, per runtime +* ``telemetry`` — the per-inference-call record written by the proxy path +* ``event_writer`` — persistence for proxy-observed events +* ``ingest`` — translation + persistence for runtime-self-reported events +* ``inference`` — the authed relay behind POST /api/agent/inference +* ``registry`` — sticky, server-authoritative A/B profile assignment +* ``lifecycle`` — aggregating a finished task's events into totals + +Note this package is a *relay and telemetry sink*, not an agent runtime. The +built-in runtime lives in the separate ``code4me2_agent`` package. +""" diff --git a/src/agents/event_writer.py b/src/agents/event_writer.py new file mode 100644 index 0000000..dfafed5 --- /dev/null +++ b/src/agents/event_writer.py @@ -0,0 +1,414 @@ +"""Persistence of agent_event rows from the proxy telemetry path. + +Both telemetry paths converge on the same table (merge decision 2), so this +module and ``agents.ingest`` are two front doors onto one +``crud.append_agent_event``. The difference is only where the data comes from: +here it's observed by the relay; there it's self-reported by the runtime. + +Every write is best-effort: a telemetry failure must never break the agent the +developer is actually using, so DB errors are logged and swallowed. +""" + +from __future__ import annotations + +import json +import logging +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional + +from agents.normalize import snippet +from agents.telemetry import InferenceRecord +from database import crud +from research.telemetry.adapters import LegacyFact, record_legacy_facts +from research.telemetry.enums import CoverageState +from research.telemetry.models import Correlations, Coverage, EventMetrics + +if TYPE_CHECKING: + from App import App + +# Source tag written to agent_event.source for rows produced by this path. +SOURCE_PROXY = "proxy" + +#: Coverage-capability marker for relay-observed provider usage. +_USAGE_CAPABILITY = "usage" + + +def _optional_int(value: object) -> Optional[int]: + try: + return int(value) if value is not None else None + except (TypeError, ValueError): + return None + + +def _model_call_fact( + record: InferenceRecord, + latency_ms: int, + span: Optional[dict], + extra: Optional[dict], +) -> LegacyFact: + """Build the canonical fact for one relay-observed model call. + + Structural metadata only: provider-token usage keeps its coverage semantics + (missing usage stays UNAVAILABLE/null, never zero) and provider-reported + values are distinguishable from byte estimates by their field names. + """ + span = span or {} + # Payload keys must be ones the privacy classifier recognises as metadata: + # an unrecognised key is treated as content (fail closed), and ingestion + # then refuses the whole event under every study that does not capture + # content (the default). Counters therefore go into ``metrics.counts``. + counts: dict[str, int] = {} + for key, value in ( + ("prompt_tokens", record.prompt_tokens), + ("completion_tokens", record.completion_tokens), + ("message_count", record.message_count), + ("tools_kept", record.tools_kept), + ("tools_stripped", record.tools_stripped), + ("step_index", span.get("step_index")), + ): + number = _optional_int(value) + if number is not None: + counts[key] = number + total = _optional_int(record.total_tokens) + # Mirror the total into counts so context accounting can sum one block + # without joining usage_tokens; absent when the producer didn't report it. + if total is not None: + counts["total_tokens"] = total + payload: dict[str, Any] = { + "model": record.model, + "finish_reason": record.finish_reason, + "upstream_status": record.upstream_status, + "request_id": record.request_id, + "agent_profile": record.agent_profile, + "call_mode": ( + None + if record.streaming is None + else ("streaming" if record.streaming else "single") + ), + } + for key in ("tool_schema_bytes", "context_window_size_bytes"): + number = _optional_int(span.get(key)) + if number is None and isinstance(extra, dict): + # The relay computes ``tool_schema_bytes`` into the ``extra`` bag. + number = _optional_int(extra.get(key)) + if number is not None: + payload[key] = number + # The rest of the free-form ``extra`` bag is deliberately NOT copied into + # the canonical payload: most of its keys (``wire_api``, + # ``openai_passthrough``, ...) do not classify as metadata, so a single one + # would get the whole event refused, and ``upstream_base_url`` is admin-only + # server configuration that must never reach researcher-visible events. The + # legacy row keeps it in ``extra_json``. Two values are carried under keys + # the classifier recognises as metadata. + if isinstance(extra, dict): + if extra.get("requested_model"): + payload["requested_model"] = extra["requested_model"] + if extra.get("wire_api"): + payload["api_kind"] = extra["wire_api"] + return LegacyFact( + kind="model_call", + occurred_at=datetime.now(timezone.utc), + payload={k: v for k, v in payload.items() if v is not None}, + metrics=EventMetrics( + usage_tokens=total, + usage_capability=Coverage( + state=( + CoverageState.AVAILABLE + if total is not None + else CoverageState.UNAVAILABLE + ), + capability=_USAGE_CAPABILITY, + ), + latency_ms=latency_ms, + counts=counts, + ), + coverage=Coverage(state=CoverageState.AVAILABLE, capability="latency"), + correlations=Correlations(correlation_id=record.request_id), + source_event_id=span.get("span_id") or record.request_id, + emitter_id="relay", + ) + + +def _tool_call_fact( + tool_execution: dict, + latency_ms: Optional[int], + parent_span_id: Optional[str], +) -> LegacyFact: + """Build the canonical fact for one relay-observed tool execution.""" + name = tool_execution.get("name") + succeeded = tool_execution.get("success", True) + arguments = tool_execution.get("arguments") + result = tool_execution.get("result") + payload: dict[str, Any] = {} + counts: dict[str, int] = {} + if name: + payload["tool_name"] = name + if isinstance(arguments, str): + # A counter, not payload: "arguments" marks a payload key as content. + counts["tool_arguments_length"] = len(arguments) + if isinstance(result, str): + # Payload for the dashboard read model, and a counter that survives a + # policy excluding agent activity (like the arguments length). + payload["tool_result_length"] = len(result) + counts["tool_result_length"] = len(result) + tool_call_id = tool_execution.get("id") or tool_execution.get("tool_call_id") + return LegacyFact( + kind="tool_call" if succeeded is not False else "tool_failed", + occurred_at=datetime.now(timezone.utc), + payload=payload, + metrics=EventMetrics(latency_ms=latency_ms, counts=counts), + coverage=Coverage(state=CoverageState.AVAILABLE, capability="tool_lifecycle"), + correlations=Correlations(tool_call_id=str(tool_call_id) if tool_call_id else None), + source_event_id=str(tool_call_id) if tool_call_id else None, + emitter_id="relay", + ) + + + +def write_model_call_event( + app: App, + task_uuid: Optional[uuid.UUID], + record: InferenceRecord, + latency_ms: int, + span: Optional[dict] = None, + extra: Optional[dict] = None, +) -> None: + """Persist one model_call event row. + + ``record`` carries the per-call fields; ``span`` carries the span/context + fields (span_id, parent_span_id, step_index, session detection state). + ``extra`` is an optional dict of non-content structural metrics with no + typed column, serialised into ``extra_json``. + + session_id / task_id are deliberately not stored on the event — they live on + ``agent_task``, and duplicating them here would let the two drift. + """ + if task_uuid is None: + return + span = span or {} + roles = record.role_breakdown or {} + parent_span_id = span.get("parent_span_id") + db = app.get_db_session() + try: + task = crud.get_agent_task(db, task_uuid) + if task is not None: + result = record_legacy_facts( + db, task=task, facts=[_model_call_fact(record, latency_ms, span, extra)] + ) + if result is not None: + # Research-bound: the canonical writer is the only authority. + # A failed canonical write is never re-recorded in the legacy + # table (ISSUE-07); it is logged and left to the producer. + if result.written: + logging.info( + f"[Agent/events] model_call canonicalized — task={task_uuid} " + f"latency_ms={latency_ms} tokens={record.total_tokens}" + ) + else: + logging.warning( + f"[Agent/events] model_call canonical write refused — " + f"task={task_uuid} reason={result.reason} retryable={result.retryable}" + ) + return + event_index = crud.reserve_agent_event_indexes(db, task_uuid, 1) + source_event_id = span.get("span_id") or record.request_id + crud.append_agent_event( + db, + task_id=task_uuid, + event_index=event_index, + event_type="model_call", + source=SOURCE_PROXY, + source_event_id=source_event_id, + ignore_duplicate_source=source_event_id is not None, + latency_ms=latency_ms, + # span identifiers + span_id=span.get("span_id") or record.request_id, + parent_span_id=_as_uuid(parent_span_id), + request_id=record.request_id, + chat_session_index=span.get("chat_session_index"), + # model_call metadata + model=record.model, + agent_profile=record.agent_profile, + streaming=record.streaming, + message_count=record.message_count, + role_system_count=roles.get("system"), + role_user_count=roles.get("user"), + role_assistant_count=roles.get("assistant"), + role_tool_count=roles.get("tool"), + tools_kept=record.tools_kept, + tools_stripped=record.tools_stripped, + tool_names_requested=record.tool_names_requested or None, + max_tokens=record.max_tokens, + prompt_tokens=record.prompt_tokens, + completion_tokens=record.completion_tokens, + total_tokens=record.total_tokens, + finish_reason=record.finish_reason, + upstream_status=record.upstream_status, + step_index=span.get("step_index"), + context_window_size_bytes=span.get("context_window_size_bytes"), + active_file=record.active_file, + first_message_hash=span.get("first_message_hash"), + chat_new_session_detected=span.get("chat_new_session_detected"), + experiment_tool_access_enabled=span.get("experiment_tool_access_enabled"), + experiment_approval_policy=span.get("experiment_approval_policy"), + # content — already nulled by the caller unless consent resolved True + first_system_message=record.first_system_message, + last_user_message=record.last_user_message, + response_text=record.response_text, + extra_json=json.dumps(extra, default=str) if extra else None, + ) + logging.info( + f"[Agent/events] model_call written — task={task_uuid} " + f"event_index={event_index} latency_ms={latency_ms} " + f"tokens={record.total_tokens}" + ) + except Exception as e: + logging.error( + f"[Agent/events] failed to write model_call for task={task_uuid} — {e}", + exc_info=True, + ) + finally: + db.close() + + +def write_tool_call_events( + app: App, + task_uuid: uuid.UUID, + chat_session_index: int, + tool_executions: list[dict], + prev_call_at: Optional[datetime] = None, + content_included: bool = False, + parent_span_id: Optional[str] = None, +) -> None: + """Persist one tool_call event per resolved tool execution. + + Third-party agents execute tools locally between inference calls, so there + are no per-tool timestamps available — only the gap between the previous + model_call and now. That gap is split evenly across the batch as an + approximate ``latency_ms``; it is an estimate, not a measurement. + + ``parent_span_id`` is the span_id of the model_call whose response requested + these tools, so each tool_call nests under it in the trace tree. Falls back + to the task root when unknown (e.g. the first call of a session). + + Tool arguments and results can contain whole file contents (a "view" result, + a patch), so they are only stored when ``content_included``; otherwise just + their lengths are recorded, which still supports "how much context did the + tool return" analysis without retaining the text. + """ + if not tool_executions: + return + + parent_span = _as_uuid(parent_span_id) or task_uuid + latency_ms: Optional[int] = None + if prev_call_at is not None: + elapsed_ms = (datetime.now() - prev_call_at).total_seconds() * 1000 + latency_ms = max(0, int(elapsed_ms / len(tool_executions))) + + db = app.get_db_session() + try: + task = crud.get_agent_task(db, task_uuid) + if task is not None: + result = record_legacy_facts( + db, + task=task, + facts=[ + _tool_call_fact(tc, latency_ms, parent_span_id) + for tc in tool_executions + ], + ) + if result is not None: + # Research-bound: never fall back to the legacy table on a + # failed canonical write (ISSUE-07). + if result.written: + logging.info( + f"[Agent/events] {len(tool_executions)} tool_call event(s) " + f"canonicalized for task={task_uuid}" + ) + else: + logging.warning( + f"[Agent/events] tool_call canonical write refused — " + f"task={task_uuid} reason={result.reason}" + ) + return + event_index = crud.reserve_agent_event_indexes(db, task_uuid, len(tool_executions)) + for offset, tc in enumerate(tool_executions): + arguments = tc.get("arguments") + result = tc.get("result") + tool_arguments: Optional[str] = None + tool_result: Optional[str] = None + tool_arguments_length: Optional[int] = None + tool_result_length: Optional[int] = None + if content_included: + tool_arguments = _as_text(arguments) + tool_result = _as_text(result) + # Lengths are structural, so record them either way — they're the + # only signal left when content storage is off. + if isinstance(arguments, str): + tool_arguments_length = len(arguments) + if isinstance(result, str): + tool_result_length = len(result) + + source_event_id = tc.get("id") or tc.get("tool_call_id") + crud.append_agent_event( + db, + task_id=task_uuid, + event_index=event_index + offset, + event_type="tool_call", + source=SOURCE_PROXY, + source_event_id=str(source_event_id) if source_event_id else None, + ignore_duplicate_source=source_event_id is not None, + latency_ms=latency_ms, + span_id=str(source_event_id) if source_event_id else str(uuid.uuid4()), + parent_span_id=parent_span, + chat_session_index=chat_session_index, + tool_name=tc.get("name"), + tool_arguments=tool_arguments, + tool_result=tool_result, + tool_arguments_length=tool_arguments_length, + tool_result_length=tool_result_length, + commit=False, + ) + db.commit() + logging.info( + f"[Agent/events] recorded {len(tool_executions)} tool_call event(s) " + f"for task={task_uuid}" + ) + except Exception as e: + db.rollback() + logging.error( + f"[Agent/events] failed writing tool_call events for task={task_uuid} " + f"— {e}", + exc_info=True, + ) + finally: + db.close() + + +def _as_uuid(value: object) -> Optional[uuid.UUID]: + """Best-effort UUID coercion for span ids arriving as strings. + + Returns None rather than raising: an unparseable span id should cost the + parent link, not the whole telemetry row. + """ + if value is None: + return None + if isinstance(value, uuid.UUID): + return value + try: + return uuid.UUID(str(value)) + except (ValueError, AttributeError, TypeError): + return None + + +def _as_text(value: object) -> Optional[str]: + """Render a tool argument/result as a truncated string for a content column.""" + if value is None: + return None + if isinstance(value, str): + return snippet(value) + try: + return snippet(json.dumps(value, default=str)) + except (TypeError, ValueError): + return snippet(str(value)) diff --git a/src/agents/inference.py b/src/agents/inference.py new file mode 100644 index 0000000..01a5966 --- /dev/null +++ b/src/agents/inference.py @@ -0,0 +1,883 @@ +"""Authed inference relay backing POST /api/agent/inference. + +This is the observation point for **third-party** agents (Goose, Codex). We +don't control their reasoning loops, so instead of asking them to report +telemetry, the plugin's local proxy transparently intercepts every inference +call they make and forwards it here (merge decision 2). That gives us: + +* per-call telemetry written into the shared ``agent_event`` table, identical in + shape to what the self-reporting ``code4me2-agent`` runtime produces; +* server-authoritative model / temperature / tool policy — the agent sends + whatever its own config says, and we override it from the profile snapshotted + onto the task, so an A/B arm actually takes effect; +* reconstruction of the tool calls the agent ran locally between requests, by + diffing the conversation against what we saw last time. + +The upstream is a *generic* OpenAI-compatible endpoint resolved from the task's +profile (merge decision 6) — Ollama, Groq, OpenRouter and OpenAI all work +through the same path. Codex's proprietary Responses API is the one special +case, handled as a normalization branch rather than the default. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import math +import os +import re +import time +import uuid +from collections import Counter +from typing import TYPE_CHECKING, AbstractSet, Any, Optional + +import httpx +from fastapi import Response +from fastapi.responses import StreamingResponse + +from agents import provider as provider_module +from agents.event_writer import write_model_call_event, write_tool_call_events +from agents.normalize import ( + extract_from_response, + extract_from_responses_api, + extract_tool_executions, + first_user_message_text, + is_meta_request, + normalize_responses_api_body, + parse_responses_api_stream, + parse_stream, + reconcile_openai_passthrough, + sanitize_schema, + snippet, + strip_info_msg, +) +from agents.telemetry import InferenceRecord +from agents.tool_call_filter import SseToolCallFilter, filter_chat_completion_body, tool_allowed +from agents.tools import KNOWN_AGENT_TOOLS +from database import crud + +if TYPE_CHECKING: + from datetime import datetime + + from research.budget.meter import InferenceMeter + + from App import App + +# Upstream request timeout. Agent turns with large contexts are slow, and a +# premature timeout looks to the developer like the agent hung. +_UPSTREAM_TIMEOUT_SECONDS = 120 + + +def _cap_responses_output_tokens(body: dict, configured_limit: str) -> Optional[int]: + """Apply an operator limit to non-OpenAI Responses requests, if configured. + + Codex may advertise a very large default output budget. Some compatible + providers reject the request before generating anything when that budget + exceeds the account's available tokens. Never raise a caller's lower cap. + """ + try: + limit = int(configured_limit) + except (TypeError, ValueError): + return None + if limit <= 0: + return None + requested = body.get("max_output_tokens") + if isinstance(requested, int) and not isinstance(requested, bool) and requested > 0: + body["max_output_tokens"] = min(requested, limit) + else: + body["max_output_tokens"] = limit + return body["max_output_tokens"] + + +def _tool_name(tool: Any) -> Optional[str]: + """The function name of one Chat Completions tool definition, if any.""" + if not isinstance(tool, dict): + return None + function = tool.get("function") + name = function.get("name") if isinstance(function, dict) else None + return name if isinstance(name, str) and name else None + + +def restrict_chat_completion_tools( + body: dict, allowlist: AbstractSet[str] +) -> tuple[list[str], list[dict]]: + """Keep only the Chat Completions tools named in ``allowlist``, in place. + + ``mcp__*`` in the allowlist admits every ``mcp__`` tool. With nothing kept, + ``tools`` and ``tool_choice`` are removed; a ``tool_choice`` that names a + removed function is dropped too, so the provider is never asked to call a + tool it was not given. Schemas are not touched. Returns the requested + names and the kept tool definitions. + """ + raw = body.get("tools") + requested = raw if isinstance(raw, list) else [] + names = [name for name in (_tool_name(tool) for tool in requested) if name] + if raw is None: + return names, [] + kept = [tool for tool in requested if tool_allowed(_tool_name(tool), allowlist)] + if kept: + body["tools"] = kept + _restrict_tool_choice(body, {_tool_name(tool) for tool in kept}) + else: + # Providers reject these without tools. + body.pop("tools", None) + body.pop("tool_choice", None) + body.pop("parallel_tool_calls", None) + return names, kept + + +def _restrict_tool_choice(body: dict, kept_names: set) -> None: + """Drop a ``tool_choice`` that names a removed tool; narrow an ``allowed_tools`` list.""" + choice = body.get("tool_choice") + if not isinstance(choice, dict): + return + if choice.get("type") == "allowed_tools": + allowed = choice.get("allowed_tools") + listed = allowed.get("tools") if isinstance(allowed, dict) else None + if isinstance(listed, list): + narrowed = [item for item in listed if _tool_name(item) in kept_names] + if narrowed: + allowed["tools"] = narrowed + else: + body.pop("tool_choice", None) + return + function = choice.get("function") + chosen = function.get("name") if isinstance(function, dict) else None + if chosen is not None and chosen not in kept_names: + body.pop("tool_choice", None) + + +def _filter_chat_completion_tools( + body: dict, profile_tools_json: Optional[str] +) -> tuple[list[str], int, int]: + """Apply a task's frozen tool selection on the agent relay (Chat Completions). + + An explicit empty profile is a no-tools arm. A missing/invalid selection + retains the legacy catalogue fallback for old profiles. Kept schemas are + sanitised for the upstream provider. + """ + raw = body.get("tools") + if not raw: + return [], 0, 0 + + profile_tools: Optional[set[str]] = None + if profile_tools_json is not None: + try: + parsed = json.loads(profile_tools_json) + if isinstance(parsed, list): + profile_tools = {str(item) for item in parsed} + except (json.JSONDecodeError, TypeError): + pass + allowlist = profile_tools if profile_tools is not None else KNOWN_AGENT_TOOLS + requested_count = len(raw) if isinstance(raw, list) else 0 + names, kept = restrict_chat_completion_tools(body, allowlist) + for tool in kept: + parameters = tool["function"].get("parameters") + if isinstance(parameters, dict): + sanitize_schema(parameters) + logging.info( + "[Agent/inference] tools — allowlist=%s requested=%s kept=%s stripped=%s", + "profile" if profile_tools is not None else "catalogue", + names, + len(kept), + requested_count - len(kept), + ) + return names, len(kept), requested_count - len(kept) + + +def _retry_after_seconds(headers: httpx.Headers, body: bytes) -> Optional[int]: + """Extract a provider rate-limit delay so agents can back off correctly.""" + header_value = headers.get("retry-after") + try: + if header_value: + return max(1, min(120, math.ceil(float(header_value)))) + except ValueError: + pass + + body_text = body.decode("utf-8", errors="replace") + match = re.search(r"try again in\s+([0-9]+(?:\.[0-9]+)?)s", body_text, re.IGNORECASE) + if match: + return max(1, min(120, math.ceil(float(match.group(1))))) + return None + + +async def run_inference( + *, + task_uuid: uuid.UUID, + session_uuid: uuid.UUID, + openai_body: dict, + enrichment: Optional[dict], + agent_profile: Optional[str], + model: str, + connection=None, + temperature: Optional[float] = None, + framework_version: Optional[str] = None, + content_included: bool = False, + profile_tools_json: Optional[str] = None, + record_observation_events: bool = True, + meter: Optional["InferenceMeter"] = None, + tool_filtering: bool = True, + response_tool_allowlist: Optional[AbstractSet[str]] = None, + app: App, +) -> Response: + """Forward one agent inference call upstream and record it as an agent_event. + + ``meter`` (research budgets) reserves the call's worst-case cost before the + body is serialised and settles/forfeits/voids the hold afterwards; without + a meter the relay behaves as before. ``tool_filtering=False`` passes the + agent's tool definitions upstream untouched (the research gateway, which + applies a release's enforced selection itself before calling this). + ``response_tool_allowlist`` also withholds, from the response, any model + call to a tool outside that selection (``agents.tool_call_filter``). + + The task and session are pre-validated by the route handler, so this + function trusts its inputs — except ``content_included``, which the route + resolves from the user's stored preference and which is the only thing + gating whether message text is persisted. + + ``connection`` is the administrator-managed provider connection resolved + from the task's frozen profile. Resolving it here (and raising + ``ProviderReadinessError`` on a missing/inactive connection or secret) is + what makes a revoked grant or rotated/missing secret block inference. + + ``enrichment`` carries plugin-supplied IDE context (active file, observed + framework version). + """ + request_id = str(uuid.uuid4()) + enrichment = enrichment or {} + + # Which wire API this request uses is read off the body, not the profile: + # the Responses API (Codex) sends `input`, Chat Completions (Goose, and the + # built-in runtime) sends `messages`. Trusting the body means a + # misconfigured profile can't send a body down the wrong normalization path. + is_responses_api = "input" in openai_body + + try: + upstream = provider_module.resolve_upstream( + model=model, + connection=connection, + framework_version=framework_version, + ) + except provider_module.ProviderReadinessError as exc: + logging.warning( + f"[Agent/inference] provider not ready ({exc.code}) for task={task_uuid}" + ) + from fastapi import HTTPException + + raise HTTPException( + status_code=503, + detail={"code": exc.code, "message": exc.message}, + ) from exc + + # Codex → OpenAI is a passthrough: the Responses API is OpenAI's own, so + # when that's where we're sending it, none of the compat rewrites apply and + # the payload goes forward essentially untouched. Against any other + # provider the proprietary fields have to be normalized away. + is_openai_upstream = "api.openai.com" in upstream.base_url + use_openai_passthrough = is_responses_api and is_openai_upstream + + requested_model = openai_body.get("model", "unknown") + + if is_responses_api and not use_openai_passthrough: + logging.info( + "[Agent/inference] Responses API → non-OpenAI upstream, normalizing" + ) + normalize_responses_api_body(openai_body) + capped = _cap_responses_output_tokens( + openai_body, os.getenv("AGENT_RESPONSES_MAX_OUTPUT_TOKENS", "") + ) + if capped is not None: + logging.info("[Agent/inference] non-OpenAI Responses output cap=%s", capped) + + # Override the model with the task's assigned one. The agent sends whatever + # its local config holds; the server decides authoritatively, from the + # profile snapshotted at task-creation time, so the A/B arm is real. + if model and model != requested_model: + logging.info( + f"[Agent/inference] model override — requested={requested_model!r} " + f"→ {model!r}" + ) + openai_body["model"] = model + elif not model: + # Defensive: task row had no model — fall back to the requested one so + # telemetry still has a concrete value to record. + model = requested_model + + if use_openai_passthrough: + reconcile_openai_passthrough(openai_body, model) + elif temperature is not None: + # Inject the profile's sampling temperature the same way as the model. + # Skipped on the OpenAI Responses passthrough, which doesn't accept it. + openai_body["temperature"] = temperature + logging.info(f"[Agent/inference] temperature override → {temperature}") + + streaming = bool(openai_body.get("stream", False)) + messages = openai_body.get("messages", []) or [] + max_tokens = openai_body.get("max_output_tokens" if is_responses_api else "max_tokens") + # The active file path is structural metadata, stored unconditionally. + active_file: Optional[str] = enrichment.get("active_file") + observed_framework: Optional[str] = enrichment.get("framework_version") + + current_messages = openai_body.get("input", []) if is_responses_api else messages + if not isinstance(current_messages, list): + current_messages = [] + + # Count prior assistant turns to derive this call's step index. + if is_responses_api: + step_index = sum( + 1 + for item in current_messages + if isinstance(item, dict) and item.get("role") == "assistant" + ) + else: + step_index = sum( + 1 for m in messages if isinstance(m, dict) and m.get("role") == "assistant" + ) + + context_window_size_bytes = len( + json.dumps(current_messages, default=str).encode("utf-8") + ) + + # Detect whether this request belongs to a new chat session within the same + # task, by comparing a fingerprint of the first message against the previous + # request's. A *changed* first message means a brand-new chat was started; + # the same first message with a shorter array is more likely context + # compaction, which is not a new session. + message_count = len(current_messages) + first_message_hash: Optional[str] = ( + hashlib.sha256( + json.dumps(current_messages[0], default=str, sort_keys=True).encode("utf-8") + ).hexdigest() + if current_messages + else None + ) + chat_session_index = 0 + new_session_detected = False + prev_message_count = 0 + prev_call_at: Optional[datetime] = None + # span_id of the previous model_call — the parent of any tool calls the + # agent has executed since, because it was that response which requested them. + prev_model_call_span_id: Optional[str] = None + approval_policy: Optional[str] = None + meta_request = is_meta_request(current_messages) + + db_lookup = app.get_db_session() + try: + last_event = crud.get_last_agent_event_for_session( + db_lookup, session_uuid, event_type="model_call" + ) + if last_event: + prev_call_at = last_event.created_at + prev_model_call_span_id = last_event.span_id + prev_first_hash = last_event.first_message_hash + chat_session_index = last_event.chat_session_index or 0 + if meta_request: + # Auxiliary request (tool-call summary / chat-title generation): + # its message array is unrelated to the main chat, so pass the + # previous session-tracking state through unchanged rather than + # treating this as a new session — or letting it mask a real one + # on the next call. + first_message_hash = prev_first_hash + message_count = last_event.message_count or message_count + prev_message_count = message_count + elif prev_first_hash is not None and prev_first_hash != first_message_hash: + new_session_detected = True + chat_session_index += 1 + else: + prev_message_count = last_event.message_count or 0 + + task = crud.get_agent_task(db_lookup, task_uuid) + if task is not None: + approval_policy = task.approval_policy + # framework_version is a per-task property. Backfill the concrete + # version the proxy observed (e.g. "goose 1.x") over the profile's + # coarse runtime name, once, rather than on every event. + if observed_framework and not task.framework_version: + crud.set_agent_task_framework_version( + db_lookup, task_uuid, observed_framework + ) + task_description = task.task_description + # task_description is the user's first message — content, so only + # derive and persist it when the user opted into content storage. + if task_description is None and content_included: + first_user_text = first_user_message_text( + current_messages, is_responses_api=is_responses_api + ) + if first_user_text: + crud.set_agent_task_description( + db_lookup, task_uuid, snippet(first_user_text) + ) + except Exception as e: + logging.warning( + f"[Agent/inference] session-state lookup failed for task={task_uuid} — {e}" + ) + finally: + db_lookup.close() + + # Tool calls the agent executed locally appear, fully resolved, in the + # messages appended since our last call. Emit one tool_call event per + # call/result pair so we don't need a reporting hook inside the agent. + if 0 <= prev_message_count <= len(current_messages): + new_messages = current_messages[prev_message_count:] + else: + new_messages = current_messages + tool_executions = extract_tool_executions( + new_messages, is_responses_api=is_responses_api + ) + if record_observation_events: + write_tool_call_events( + app, + task_uuid, + chat_session_index, + tool_executions, + prev_call_at, + content_included, + parent_span_id=prev_model_call_span_id, + ) + + system_content = next( + ( + m.get("content", "") + for m in messages + if isinstance(m, dict) and m.get("role") == "system" + ), + "", + ) + + original_tools = openai_body.get("tools", []) or [] + # Whether this condition has any tools available at all — distinguishes a + # deliberately tool-free arm from one with a non-empty allowlist. + experiment_tool_access_enabled = bool(original_tools) + tool_names_requested = [ + name for name in (_tool_name(tool) for tool in original_tools) if name + ] + tools_kept = 0 + tools_stripped = 0 + + if original_tools and not is_responses_api and tool_filtering: + # Tool filtering and schema sanitisation apply only to Chat Completions. + # Codex manages its own tool schemas for the Responses API — those are + # passed through as-is (already normalized above where needed). + # + tool_names_requested, tools_kept, tools_stripped = _filter_chat_completion_tools( + openai_body, profile_tools_json + ) + + # stream_options.include_usage is a Chat Completions extension and the only + # way to get token counts out of a stream; the Responses API reports usage + # in its completion event instead. + if streaming and not is_responses_api: + openai_body.setdefault("stream_options", {})["include_usage"] = True + + # Reasoning models add this to assistant messages, but non-reasoning models + # reject it with a 400 invalid_request_error. Since the server may override + # the model to a non-reasoning one, strip it defensively. + if "messages" in openai_body: + for msg in openai_body["messages"]: + if isinstance(msg, dict) and msg.get("role") == "assistant": + msg.pop("reasoning_content", None) + + # Research budgets: the body is final here (model, temperature, tools and + # stream options applied), so the hold estimate sees exactly what goes + # upstream. A refusal (402 budget, 503 price missing) is returned before + # anything is sent; on success the meter has capped max_tokens in place. + if meter is not None: + refusal = await meter.reserve( + openai_body, request_id=request_id, upstream_base_url=upstream.base_url + ) + if refusal is not None: + logging.info( + f"[Agent/inference] refused by budget meter request_id={request_id} " + f"task={task_uuid} status={refusal.status_code}" + ) + return refusal + max_tokens = openai_body.get("max_tokens") or openai_body.get("max_completion_tokens") + + # Schema size is structural telemetry. Token count is estimated in the + # analytics layer because providers use different tokenizers. + tool_schema_bytes = len( + json.dumps(openai_body.get("tools", []), default=str).encode("utf-8") + ) + body_bytes = json.dumps(openai_body).encode() + role_counts = Counter( + m.get("role", "unknown") for m in messages if isinstance(m, dict) + ) + last_user = next( + ( + m.get("content", "") + for m in reversed(messages) + if isinstance(m, dict) and m.get("role") == "user" + ), + None, + ) + last_user = ( + strip_info_msg(last_user) or None if isinstance(last_user, str) else None + ) + + # Conversation content is only persisted when the user opted in; otherwise + # these stay null and only the structural columns are written. + stored_system_message = (system_content or None) if content_included else None + stored_last_user = last_user if content_included else None + + span = { + "span_id": request_id, + "parent_span_id": str(task_uuid), + "step_index": step_index, + "context_window_size_bytes": context_window_size_bytes, + "first_message_hash": first_message_hash, + "chat_session_index": chat_session_index, + "chat_new_session_detected": new_session_detected, + "experiment_tool_access_enabled": experiment_tool_access_enabled, + "experiment_approval_policy": approval_policy, + } + # Non-content structural extras with no typed column of their own. + extra = { + "wire_api": "responses" if is_responses_api else "chat_completions", + "upstream_base_url": upstream.base_url, + "openai_passthrough": use_openai_passthrough, + "meta_request": meta_request, + "requested_model": requested_model, + "tool_schema_bytes": tool_schema_bytes, + } + + upstream_url = upstream.endpoint(responses_api=is_responses_api) + upstream_headers = { + **provider_module.upstream_request_headers(upstream, session_key=str(session_uuid)), + "Content-Type": "application/json", + } + + logging.info( + f"[Agent/inference] start request_id={request_id} task={task_uuid} " + f"model={model} stream={streaming} messages={len(messages)} " + f"max_tokens={max_tokens} → {upstream_url} body={len(body_bytes)}B" + ) + + t0 = time.monotonic() + + def _build_record( + prompt_tok: Optional[int], + completion_tok: Optional[int], + total_tok: Optional[int], + finish_reason: Optional[str], + response_text: Optional[str], + latency_ms: int, + upstream_status: int, + ) -> InferenceRecord: + return InferenceRecord( + request_id=request_id, + session_id=str(session_uuid), + task_id=str(task_uuid), + agent_profile=agent_profile, + model=model, + streaming=streaming, + message_count=len(messages), + role_breakdown=dict(role_counts), + first_system_message=stored_system_message, + last_user_message=stored_last_user, + tools_kept=tools_kept, + tools_stripped=tools_stripped, + tool_names_requested=tool_names_requested, + max_tokens=max_tokens, + active_file=active_file, + prompt_tokens=prompt_tok, + completion_tokens=completion_tok, + total_tokens=total_tok, + finish_reason=finish_reason, + response_text=response_text if content_included else None, + latency_ms=latency_ms, + upstream_status=upstream_status, + ) + + def _upstream_unreachable(error: Exception) -> Response: + """502 for a provider we could not reach; the failure is still a finding.""" + latency_ms = int((time.monotonic() - t0) * 1000) + logging.error( + f"[Agent/inference] upstream unreachable request_id={request_id} " + f"url={upstream_url}: {type(error).__name__}: {error}" + ) + record = _build_record(None, None, None, "upstream_unreachable", None, latency_ms, 502) + _log_record(record) + if record_observation_events: + write_model_call_event(app, task_uuid, record, latency_ms, span, extra) + body = { + "error": { + "message": ( + "The model provider could not be reached " + f"({type(error).__name__}). Try again shortly." + ), + "type": "upstream_unavailable", + "code": 502, + } + } + return Response( + content=json.dumps(body), + status_code=502, + media_type="application/json", + ) + + if streaming: + sse_buffer: list[bytes] = [] + stream_aborted: list[bool] = [False] + tool_call_filter = ( + SseToolCallFilter(response_tool_allowlist) + if response_tool_allowlist is not None + else None + ) + + stream_client = httpx.AsyncClient(timeout=_UPSTREAM_TIMEOUT_SECONDS) + stream_request = stream_client.build_request( + "POST", + upstream_url, + content=body_bytes, + headers=upstream_headers, + ) + try: + upstream_stream = await stream_client.send(stream_request, stream=True) + except httpx.HTTPError as error: + await stream_client.aclose() + if meter is not None: + await meter.resolve_transport_error(error) + return _upstream_unreachable(error) + + # Do not turn an upstream 429 into a successful-looking SSE response. + # Goose retries when it sees the status and Retry-After; yielding the + # error body through StreamingResponse would otherwise default to 200. + if upstream_stream.status_code >= 400: + error_body = await upstream_stream.aread() + retry_after = _retry_after_seconds(upstream_stream.headers, error_body) + await upstream_stream.aclose() + await stream_client.aclose() + if meter is not None: + # The provider refused before generating: nothing was billed. + await meter.resolve_upstream_error(upstream_stream.status_code) + latency_ms = int((time.monotonic() - t0) * 1000) + record = _build_record(None, None, None, None, None, latency_ms, upstream_stream.status_code) + _log_record(record) + if record_observation_events: + write_model_call_event(app, task_uuid, record, latency_ms, span, extra) + response_headers = {} + if retry_after is not None: + response_headers["Retry-After"] = str(retry_after) + return Response( + content=error_body, + status_code=upstream_stream.status_code, + headers=response_headers, + media_type=upstream_stream.headers.get("content-type", "application/json"), + ) + + stream_timed_out: list[bool] = [False] + stream_timeout = ( + meter.settings.stream_total_timeout_seconds if meter is not None else None + ) + + async def _stream(): + try: + logging.info( + f"[Agent/inference] ← upstream " + f"status={upstream_stream.status_code} (stream)" + ) + # A metered stream may not outlive its reservation deadline: the + # wall-clock cap (checked per chunk; each read is already bounded + # by the httpx timeout) turns a hung provider into a forfeit + # instead of an open hold. No cancel scope wraps the yields, so + # a generator finalised from another task after a client + # disconnect still settles cleanly in `finally`. + async for chunk in upstream_stream.aiter_bytes(): + if stream_timeout is not None and (time.monotonic() - t0) > stream_timeout: + stream_aborted[0] = True + stream_timed_out[0] = True + logging.warning( + f"[Agent/inference] stream exceeded {stream_timeout}s — " + f"request_id={request_id}" + ) + break + sse_buffer.append(chunk) + # The agent gets the filtered stream; usage and settlement + # still read the provider's original bytes. + delivered = tool_call_filter.feed(chunk) if tool_call_filter else chunk + if delivered: + yield delivered + if tool_call_filter is not None: + rest = tool_call_filter.flush() + if rest: + yield rest + except Exception as e: + stream_aborted[0] = True + logging.warning(f"[Agent/inference] stream aborted mid-flight — {e}") + finally: + await upstream_stream.aclose() + await stream_client.aclose() + if tool_call_filter is not None and tool_call_filter.withheld: + _log_withheld_tool_calls(request_id, tool_call_filter.withheld) + if tool_call_filter is not None and tool_call_filter.dropped: + logging.warning( + "[Agent/inference] dropped %d stream line(s) the tool filter could not judge " + "request_id=%s", + tool_call_filter.dropped, + request_id, + ) + # Telemetry is written in `finally` so an aborted stream still + # produces a row — a dropped connection is itself a finding. + latency_ms = int((time.monotonic() - t0) * 1000) + raw_sse = b"".join(sse_buffer).decode("utf-8", errors="replace") + parser = parse_responses_api_stream if is_responses_api else parse_stream + ( + prompt_tok, + completion_tok, + total_tok, + finish_reason, + response_text, + ) = parser(raw_sse) + if stream_aborted[0]: + finish_reason = "stream_aborted" + + record = _build_record( + prompt_tok, + completion_tok, + total_tok, + finish_reason, + response_text, + latency_ms, + upstream_stream.status_code, + ) + _log_record(record) + if record_observation_events: + write_model_call_event( + app, task_uuid, record, latency_ms, span, extra + ) + # Budget settlement runs whatever ended the stream (completion, + # upstream read error, client disconnect via aclose(), shutdown): + # a stream without a usage chunk forfeits its hold. The meter + # shields this await from cancellation and never raises. + if meter is not None: + if stream_timed_out[0]: + await meter.resolve_timeout() + else: + await meter.resolve_stream( + raw_sse, upstream_status=upstream_stream.status_code + ) + + return StreamingResponse( + _stream(), + media_type="text/event-stream", + headers=meter.response_headers() if meter is not None else None, + ) + + try: + async with httpx.AsyncClient(timeout=_UPSTREAM_TIMEOUT_SECONDS) as client: + upstream_resp = await client.post( + upstream_url, content=body_bytes, headers=upstream_headers + ) + except httpx.HTTPError as error: + if meter is not None: + await meter.resolve_transport_error(error) + return _upstream_unreachable(error) + + latency_ms = int((time.monotonic() - t0) * 1000) + + if upstream_resp.status_code >= 400: + logging.error( + f"[Agent/inference] upstream error {upstream_resp.status_code}: " + f"{upstream_resp.text}" + ) + + resp_json = None + try: + resp_json = upstream_resp.json() + except Exception as e: # noqa: BLE001 - an unparseable body is handled below + logging.warning(f"[Agent/inference] could not parse upstream response — {e}") + + if meter is not None: + if upstream_resp.status_code >= 400: + await meter.resolve_upstream_error(upstream_resp.status_code) + else: + await meter.resolve_response(resp_json, upstream_status=upstream_resp.status_code) + + prompt_tok = completion_tok = total_tok = finish_reason = response_text = None + try: + extractor = ( + extract_from_responses_api if is_responses_api else extract_from_response + ) + if resp_json is not None: + prompt_tok, completion_tok, total_tok, finish_reason, response_text = extractor( + resp_json + ) + except Exception as e: + # A body we can't parse costs telemetry detail, not the response — the + # agent still gets whatever the upstream said. + logging.warning(f"[Agent/inference] could not parse upstream response — {e}") + + record = _build_record( + prompt_tok, + completion_tok, + total_tok, + finish_reason, + response_text, + latency_ms, + upstream_resp.status_code, + ) + _log_record(record) + if record_observation_events: + write_model_call_event(app, task_uuid, record, latency_ms, span, extra) + + response_headers = {} + if upstream_resp.status_code == 429: + retry_after = _retry_after_seconds(upstream_resp.headers, upstream_resp.content) + if retry_after is not None: + response_headers["Retry-After"] = str(retry_after) + if meter is not None: + response_headers.update(meter.response_headers()) + + content = upstream_resp.content + status_code = upstream_resp.status_code + if response_tool_allowlist is not None and status_code < 400: + if not isinstance(resp_json, dict): + # A body nothing here can read could carry a call nothing here can + # judge: an enforced selection fails closed. + logging.warning( + f"[Agent/inference] unreadable upstream body withheld request_id={request_id}" + ) + content = json.dumps({"error": { + "message": "The model provider returned a response the study could not check.", + "type": "upstream_unreadable", "code": 502, + }}).encode() + status_code = 502 + else: + withheld = filter_chat_completion_body(resp_json, response_tool_allowlist) + if withheld: + _log_withheld_tool_calls(request_id, withheld) + content = json.dumps(resp_json).encode() + + return Response( + content=content, + status_code=status_code, + headers=response_headers, + media_type="application/json", + ) + + +def _log_withheld_tool_calls(request_id: str, names: list[str]) -> None: + """A model called a tool outside the enforced selection; the agent never saw it.""" + logging.warning( + "[Agent/inference] withheld %d tool call(s) outside the enforced selection " + "request_id=%s tools=%s", + len(names), + request_id, + sorted(set(names)), + ) + + +def _log_record(record: InferenceRecord) -> None: + status_flag = ( + "OK" if (record.upstream_status or 0) < 400 else f"ERR{record.upstream_status}" + ) + logging.info( + f"[Agent/inference] done request_id={record.request_id} " + f"task_id={record.task_id!r} {status_flag} model={record.model} " + f"stream={record.streaming} " + f"tokens(in/out/tot)={record.prompt_tokens}/{record.completion_tokens}/" + f"{record.total_tokens} finish={record.finish_reason} " + f"latency_ms={record.latency_ms} " + f"tools(kept/stripped)={record.tools_kept}/{record.tools_stripped}" + ) diff --git a/src/agents/ingest.py b/src/agents/ingest.py new file mode 100644 index 0000000..8f438de --- /dev/null +++ b/src/agents/ingest.py @@ -0,0 +1,580 @@ +"""Ingestion of self-reported telemetry from the built-in code4me2-agent runtime. + +This is the other half of merge decision 2 ("hybrid telemetry, per agent type"). +Because we *own* the ``code4me2-agent`` ReAct loop, it can report its own steps +rather than being observed from outside — which yields strictly better data than +proxy interception (it sees iterations, denials and parse failures that never +become HTTP requests). + +Both paths land in the same ``agent_event`` table, so downstream analysis never +has to care which runtime produced a row. This module is the translation layer: +it maps the runtime's event envelope onto the columnar schema, promoting known +fields to typed columns and routing the rest into the JSON overflow column. + +Two envelope quirks it reconciles: + +* The runtime reports a model call as *two* events (``agent.model.requested`` + then ``agent.model.completed``) and a tool call as two more + (``agent.tool.called`` / ``agent.tool.completed``), because it emits as it + goes. The proxy path writes one row per call. To keep the two paths + comparable, the request-side fields are merged forward onto the completion + row (matched by ``request_id`` / ``tool_call_id``), while the request events + are still kept as their own low-value rows so a crashed run isn't silently + lost. +* The envelope carries a client-side ``privacy`` block including + ``raw_capture_enabled``. That is **descriptive only** and is never used to + decide what gets stored — per merge decision 3 the consent check is + server-side (``resolve_store_agent_content``). The claimed block is retained + in ``extra_json`` purely so a mismatch between what the client thought and + what the server did is auditable. +""" + +from __future__ import annotations + +import json +import logging +import uuid +from datetime import datetime, timezone +from typing import Any, Optional + +from sqlalchemy.orm import Session + +from database import crud +from research.telemetry.adapters import ( + CanonicalIngestionFailed, + LegacyFact, + record_legacy_facts, + existing_source_event_ids, + research_bound, +) +from research.telemetry.enums import CoverageState +from research.telemetry.models import Correlations, Coverage, EventMetrics + +# Source tag written to agent_event.source for rows produced by this path. +SOURCE_SELF_REPORT = "code4me2_agent" + +# Runtime event_type → unified agent_event.event_type. +# +# `model_call` and `tool_call` are the two types that carry typed columns and +# feed task aggregation, so only the *completion* events map onto them; their +# request-side counterparts get distinct types to avoid double-counting tokens +# and steps in `lifecycle.finalize_agent_task`. +EVENT_TYPE_MAP: dict[str, str] = { + "agent.run.started": "run_started", + "agent.run.completed": "run_completed", + "agent.model.requested": "model_request", + "agent.model.completed": "model_call", + "agent.tool.called": "tool_request", + "agent.tool.completed": "tool_call", + "agent.tool.denied": "tool_denied", + "agent.tool.failed": "tool_failed", + "agent.permission.requested": "permission_requested", + "agent.permission.decided": "permission_decided", + "agent.adapter.loop_failed": "error", + "agent.adapter.parse_failed": "error", + "agent.request.received": "observation", + "agent.response.completed": "observation", +} + +# Metrics keys promoted out of the free-form `metrics` block into typed columns. +# Anything not listed stays in extra_json. +_PROMOTED_METRIC_KEYS = frozenset( + {"duration_ms", "prompt_tokens", "completion_tokens", "total_tokens"} +) + +# Keys ``map_event_to_columns`` carries only for the canonical fact; the legacy +# ``agent_event`` table has no column for them. +_CANONICAL_ONLY_COLUMNS = frozenset( + {"run_id", "message_id", "tool_call_id", "decision", "decision_scope", "tool_kind"} +) + +# Self-reported "Revise…" decision keys kept outside the content blob: the +# strings go to the canonical payload, the counts to ``metrics.counts``. +_REVISE_TEXT_KEYS = ("elicitation_action", "revise_status") +_REVISE_COUNT_KEYS = ("hunk_count", "kept_hunk_count") + + +def map_event_type(runtime_event_type: str) -> str: + """Translate a runtime event type, defaulting unknown ones to `observation`. + + Unknown types are kept rather than rejected: the runtime evolves faster than + this map, and a new event name should degrade to a generic row with its + payload intact, not 400 the whole batch. + """ + mapped = EVENT_TYPE_MAP.get(runtime_event_type) + if mapped is None: + logging.info( + f"[Agent/ingest] unmapped event_type {runtime_event_type!r} " + f"— storing as 'observation'" + ) + return "observation" + return mapped + + +def _as_int(value: Any) -> Optional[int]: + """Coerce a reported value to int, or None if it isn't numeric.""" + if value is None or isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _as_uuid(value: Any) -> Optional[uuid.UUID]: + """Best-effort UUID coercion. The runtime mints ids as `uuid4().hex`, which + parses fine; anything else degrades to None rather than failing the row.""" + if value is None: + return None + try: + return uuid.UUID(str(value)) + except (ValueError, AttributeError, TypeError): + return None + + +def _parse_timestamp(value: Any) -> Optional[datetime]: + """Parse the runtime's ISO-8601 timestamp (it emits a trailing 'Z').""" + if isinstance(value, datetime): + return value + if not isinstance(value, str) or not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def _text_or_json(value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, str): + return value + try: + return json.dumps(value, default=str) + except (TypeError, ValueError): + return str(value) + + +def build_correlation_index(events: list[dict]) -> tuple[dict, dict]: + """Pre-scan a batch to collect request-side fields for forward-merging. + + Returns ``(by_request_id, by_tool_call_id)``. Only fields that belong on the + completion row are collected, so the merge can't overwrite anything the + completion event reports itself. + """ + by_request_id: dict[str, dict] = {} + by_tool_call_id: dict[str, dict] = {} + + for event in events: + event_type = event.get("event_type") + payload = event.get("payload") or {} + if not isinstance(payload, dict): + continue + + if event_type == "agent.model.requested": + request_id = event.get("request_id") + if request_id: + by_request_id[str(request_id)] = { + "message_count": _as_int(payload.get("message_count")), + # The ReAct loop's iteration number is this call's step index. + "step_index": _as_int(payload.get("iteration")), + "approx_token_count": _as_int(payload.get("approx_token_count")), + } + elif event_type == "agent.tool.called": + tool_call_id = payload.get("tool_call_id") + if tool_call_id: + by_tool_call_id[str(tool_call_id)] = { + "arguments": payload.get("arguments"), + } + + return by_request_id, by_tool_call_id + + +def map_event_to_columns( + event: dict, + *, + content_included: bool, + by_request_id: dict, + by_tool_call_id: dict, +) -> dict: + """Translate one runtime event envelope into ``crud.append_agent_event`` kwargs. + + ``content_included`` is the server-resolved consent decision. When it is + False every content-bearing field (payload, raw_payload, tool arguments and + results, response text) is dropped here, before it can reach the database — + the structural columns and the non-content ``extra_json`` are unaffected. + """ + runtime_event_type = str(event.get("event_type") or "") + event_type = map_event_type(runtime_event_type) + payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} + metrics = event.get("metrics") if isinstance(event.get("metrics"), dict) else {} + request_id = event.get("request_id") + + # Forward-merge the request-side fields onto the completion row so a + # self-reported model_call has the same populated columns as a proxied one. + merged: dict = {} + if event_type == "model_call" and request_id: + merged = by_request_id.get(str(request_id), {}) + tool_arguments_value: Any = None + if event_type in ("tool_call", "tool_denied", "tool_failed"): + tool_call_id = payload.get("tool_call_id") + if tool_call_id: + tool_arguments_value = by_tool_call_id.get(str(tool_call_id), {}).get( + "arguments" + ) + + usage = payload.get("usage") if isinstance(payload.get("usage"), dict) else {} + + # latency: metrics.duration_ms is authoritative; tool events report it in + # their payload instead. + latency_ms = _as_int(metrics.get("duration_ms")) + if latency_ms is None: + latency_ms = _as_int(payload.get("duration_ms")) + + tool_result_value = payload.get("result") if "result" in payload else None + tool_arguments_text = _text_or_json(tool_arguments_value) + tool_result_text = _text_or_json(tool_result_value) + + # Non-content structural leftovers. `observability` and `privacy` are the + # runtime's own descriptive blocks; the un-promoted metrics keys are + # adapter-specific counters that don't warrant a column each. + extra: dict[str, Any] = { + "runtime_event_type": runtime_event_type, + "sequence": event.get("sequence"), + "message_id": event.get("message_id"), + } + leftover_metrics = { + k: v for k, v in metrics.items() if k not in _PROMOTED_METRIC_KEYS + } + if leftover_metrics: + extra["metrics"] = leftover_metrics + if isinstance(event.get("observability"), dict): + extra["observability"] = event["observability"] + if isinstance(event.get("privacy"), dict): + # Retained for audit only — the storage decision was made server-side. + extra["client_claimed_privacy"] = event["privacy"] + if merged.get("approx_token_count") is not None: + extra["approx_token_count"] = merged["approx_token_count"] + for key in ("status", "backend_type", "failure_reason", "denial_reason", "path"): + if payload.get(key) is not None: + extra[key] = payload[key] + # Why the runtime made a model call ("turn", "summarize", "self_review"): + # structural, so analytics can separate harness calls from the agent's own + # steps even when content is not stored. + if event_type == "model_call" and isinstance(payload.get("call_purpose"), str): + extra["call_purpose"] = payload["call_purpose"][:32] + # A "Revise…" decision: the form outcome and what became of the kept hunks + # (BEHAVIORAL) and the hunk counts (SYSTEM) are structural; the user's + # instructions (``text``) stay in the content blob. + for key in _REVISE_TEXT_KEYS: + if isinstance(payload.get(key), str): + extra[key] = payload[key][:32] + for key in _REVISE_COUNT_KEYS: + count = _as_int(payload.get(key)) + if count is not None: + extra[key] = count + + # Content: the payload may quote user prompts, model output or file + # contents, and raw_payload is the unredacted form of the same. Both are + # dropped wholesale without consent. + payload_json: Optional[str] = None + if content_included: + content_blob: dict[str, Any] = {"payload": payload} + if event.get("raw_payload") is not None: + content_blob["raw_payload"] = event["raw_payload"] + payload_json = _text_or_json(content_blob) + + return { + "event_type": event_type, + "source": SOURCE_SELF_REPORT, + "schema_version": event.get("schema_version"), + "run_id": event.get("run_id"), + "message_id": event.get("message_id"), + "occurred_at": _parse_timestamp(event.get("timestamp")), + "latency_ms": latency_ms, + # The runtime's own event id becomes the span id, which is what makes + # re-ingestion of a retried batch detectable. + "span_id": str(event["event_id"]) if event.get("event_id") else None, + "parent_span_id": _as_uuid(event.get("parent_event_id")), + "request_id": str(request_id) if request_id else None, + # tool_call identity for tool/permission events (structural ids). + "tool_call_id": ( + str(payload.get("tool_call_id")) + if payload.get("tool_call_id") is not None + else None + ), + # permission decision metadata: outcome and scope are structural + # (BEHAVIORAL-classified), never content, so they persist under + # metadata-only policies. + "decision": payload.get("decision"), + "decision_scope": payload.get("decision_scope"), + "tool_kind": payload.get("kind"), + # model_call fields + "model": payload.get("model"), + "message_count": merged.get("message_count"), + "step_index": merged.get("step_index"), + "prompt_tokens": _as_int( + metrics.get("prompt_tokens") or usage.get("prompt_tokens") + ), + "completion_tokens": _as_int( + metrics.get("completion_tokens") or usage.get("completion_tokens") + ), + "total_tokens": _as_int( + metrics.get("total_tokens") or usage.get("total_tokens") + ), + "finish_reason": payload.get("finish_reason"), + # tool_call fields + "tool_name": payload.get("tool_name"), + "tool_arguments": tool_arguments_text if content_included else None, + "tool_result": tool_result_text if content_included else None, + # Lengths are structural — recorded regardless of consent, so "how much + # context did this tool return" stays answerable either way. + "tool_arguments_length": ( + len(tool_arguments_text) if tool_arguments_text is not None else None + ), + "tool_result_length": ( + len(tool_result_text) if tool_result_text is not None else None + ), + # JSON overflow + "extra_json": json.dumps(extra, default=str), + "payload_json": payload_json, + } + + +def _extra_value(extra_json: object, key: str) -> object: + if not isinstance(extra_json, str): + return None + try: + extra = json.loads(extra_json) + except ValueError: + return None + return extra.get(key) if isinstance(extra, dict) else None + + +def _fact_from_columns(columns: dict, *, content_included: bool = False) -> "LegacyFact": + """Build the canonical fact for one self-reported event. + + Structural metadata is always included. When the study policy allowed + content, the mapped content columns are carried in the canonical payload + under content-classified keys, so the ingestion privacy gate is the final + authority before persistence (ISSUE-01). + """ + kind = columns.get("event_type") or "observation" + counts: dict[str, int] = {} + for key in ("prompt_tokens", "completion_tokens", "message_count", "step_index"): + value = columns.get(key) + if value is not None: + counts[key] = int(value) + for key in ("tool_arguments_length", "tool_result_length"): + value = columns.get(key) + if value is not None: + counts[key] = int(value) + total = columns.get("total_tokens") + if total is not None: + counts["total_tokens"] = int(total) + for key in _REVISE_COUNT_KEYS: + value = _as_int(_extra_value(columns.get("extra_json"), key)) + if value is not None: + counts[key] = value + payload: dict[str, Any] = {} + for key in ("model", "finish_reason", "tool_name", "request_id"): + value = columns.get(key) + if value is not None: + payload[key] = value + # Permission decision metadata (structural, never content). + for key in ("tool_kind", "decision", "decision_scope"): + value = columns.get(key) + if value is not None: + payload[key] = value + call_purpose = _extra_value(columns.get("extra_json"), "call_purpose") + if isinstance(call_purpose, str): + # BEHAVIORAL-classified ("call" token): kept or dropped by the study + # policy like other structural metadata, never refused as content. + payload["call_purpose"] = call_purpose + for key in _REVISE_TEXT_KEYS: + # BEHAVIORAL ("action"/"status" tokens), like the decision itself. + value = _extra_value(columns.get("extra_json"), key) + if isinstance(value, str): + payload[key] = value + if content_included: + if columns.get("tool_arguments") is not None: + payload["tool_arguments"] = columns["tool_arguments"] + if columns.get("tool_result") is not None: + # ``result_text`` carries a content token so the privacy gate + # classifies it as CONTENT (``tool_result`` alone would not). + payload["result_text"] = columns["tool_result"] + if columns.get("payload_json") is not None: + payload["payload"] = columns["payload_json"] + return LegacyFact( + kind=kind, + occurred_at=columns.get("occurred_at") or datetime.now(timezone.utc), + payload=payload, + metrics=EventMetrics( + usage_tokens=int(total) if total is not None else None, + usage_capability=Coverage( + state=( + CoverageState.AVAILABLE + if total is not None + else CoverageState.UNAVAILABLE + ), + capability="usage", + reason=( + None + if total is not None + else "usage not reported by producer" + ), + ), + latency_ms=columns.get("latency_ms"), + counts=counts, + ), + coverage=Coverage(state=CoverageState.AVAILABLE, capability="self_report"), + correlations=Correlations( + correlation_id=columns.get("request_id"), + tool_call_id=columns.get("tool_call_id"), + message_id=columns.get("message_id"), + # The run scopes the trace; the runtime's own event id is the + # span and its reported parent (if any) the parent link. The + # request id links one model invocation with the tools reported + # under it; tools without one keep a null link rather than an + # invented one. + trace_id=columns.get("run_id"), + span_id=columns.get("span_id"), + parent_span_id=( + str(columns["parent_span_id"]) + if columns.get("parent_span_id") is not None + else None + ), + model_call_id=columns.get("request_id"), + ), + source_event_id=columns.get("span_id"), + emitter_id="self-report", + ) + + +def ingest_event_batch( + db: Session, + *, + task_id: uuid.UUID, + events: list[dict], + content_included: bool, + agent_profile: Optional[str] = None, +) -> tuple[int, list[str]]: + """Persist a batch of self-reported events against ``task_id``. + + Returns ``(ingested_count, skipped_span_ids)``. Already-seen event ids are + skipped rather than erroring, which makes the endpoint safe for the runtime + to retry after a network failure — the uploader has no way to know whether a + timed-out POST was applied. + + The whole batch commits once. Source identity makes retries idempotent; + task-local indexes are allocated atomically, so concurrent batches cannot + claim the same event position. + """ + if not events: + return 0, [] + + # Sort by the runtime's own sequence so event_index reflects real ordering + # even if the batch arrives out of order. + ordered = sorted(events, key=lambda e: e.get("sequence") or 0) + + source_event_ids = [str(e["event_id"]) for e in ordered if e.get("event_id")] + already_seen = set( + crud.find_existing_agent_event_source_ids( + db, + task_id=task_id, + source=SOURCE_SELF_REPORT, + source_event_ids=source_event_ids, + ) + ) + task = crud.get_agent_task(db, task_id) + if task is not None and research_bound(task): + # Research-bound tasks are written to the canonical store only, so a + # retried batch must be checked there or every event is stored twice. + already_seen |= existing_source_event_ids(db, task, source_event_ids) + + by_request_id, by_tool_call_id = build_correlation_index(ordered) + skipped: list[str] = [] + pending: list[tuple[dict, str]] = [] + seen_in_batch: set[str] = set() + for event in ordered: + source_event_id = str(event["event_id"]) if event.get("event_id") else None + if source_event_id is None: + continue + if source_event_id in already_seen or source_event_id in seen_in_batch: + skipped.append(source_event_id) + continue + seen_in_batch.add(source_event_id) + pending.append((event, source_event_id)) + + first_index = ( + crud.reserve_agent_event_indexes(db, task_id, len(pending)) if pending else 0 + ) + # The canonical adapter is tried first: a research-bound task's events are + # persisted through the one ingestion writer and never also dual-written to + # the legacy table. A task with no research binding keeps the legacy path. + if pending: + facts = [] + for event, source_event_id in pending: + columns = map_event_to_columns( + event, + content_included=content_included, + by_request_id=by_request_id, + by_tool_call_id=by_tool_call_id, + ) + facts.append( + _fact_from_columns(columns, content_included=content_included) + ) + if task is not None: + result = record_legacy_facts( + db, task=task, facts=facts, first_sequence=first_index + ) + if result is not None: + # Research-bound task: the canonical writer is the only + # authority. Never fall back to the legacy table (ISSUE-07). + if not result.written: + # Ids and reason codes only (never payloads): why the batch + # was refused is otherwise invisible in the log. + logging.warning( + f"[Agent/ingest] canonical ingestion refused the batch for task {task_id}: " + f"reason={result.reason} ack={result.ack}" + ) + raise CanonicalIngestionFailed( + reason=result.reason, + message=result.message, + retryable=result.retryable, + ack=result.ack, + ) + return len(facts), skipped + ingested = 0 + for offset, (event, source_event_id) in enumerate(pending): + columns = map_event_to_columns( + event, + content_included=content_included, + by_request_id=by_request_id, + by_tool_call_id=by_tool_call_id, + ) + inserted = crud.append_agent_event( + db, + task_id=task_id, + event_index=first_index + offset, + agent_profile=agent_profile, + source_event_id=source_event_id, + ignore_duplicate_source=True, + commit=False, + **{ + key: value + for key, value in columns.items() + if key not in _CANONICAL_ONLY_COLUMNS + }, + ) + if inserted is None: + skipped.append(source_event_id) + else: + ingested += 1 + + db.commit() + logging.info( + f"[Agent/ingest] task={task_id} ingested={ingested} " + f"skipped_duplicates={len(skipped)} content={content_included}" + ) + return ingested, skipped diff --git a/src/agents/lifecycle.py b/src/agents/lifecycle.py new file mode 100644 index 0000000..8f047f4 --- /dev/null +++ b/src/agents/lifecycle.py @@ -0,0 +1,134 @@ +"""Finalizing an AgentTask: aggregating its events into task-level totals. + +Called from three places: + +* ``POST /api/agent/task/{task_id}/close`` — the plugin closes the previous + session's task on startup, because it cannot reliably detect when an ACP + agent process exits; +* session deactivation — closes any tasks left open when the user logs out; +* ``POST /api/agent/events/ingest`` — when a self-reporting runtime sends its + ``run_completed`` event. +""" + +from __future__ import annotations + +import logging +import uuid +from datetime import datetime +from typing import Optional + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from database import crud + + +def _canonical_totals(db: Session, task_id: uuid.UUID) -> Optional[tuple[int, int, int]]: + """``(steps, prompt_tokens, completion_tokens)`` from ``research_event``, or None. + + Steps are the relay/self-report facts (rows stamped with ``legacy_kind``, + the marker the canonical adapters attach); token totals come from the + ``model_call`` facts' metrics. Advisory: any failure leaves the legacy totals. + """ + task = crud.get_agent_task(db, task_id) + run_id = getattr(task, "external_run_id", None) if task is not None else None + if not run_id: + return None + begin_nested = getattr(db, "begin_nested", None) + savepoint = begin_nested() if callable(begin_nested) else None + try: + row = db.execute( + text( + """ + SELECT + COUNT(*) FILTER ( + WHERE e.envelope_json -> 'payload' ->> 'legacy_kind' IS NOT NULL + ) AS steps, + COALESCE(SUM( + (e.envelope_json -> 'metrics' -> 'counts' ->> 'prompt_tokens')::bigint + ) FILTER ( + WHERE e.envelope_json -> 'payload' ->> 'legacy_kind' = 'model_call' + ), 0) AS prompt_tokens, + COALESCE(SUM( + (e.envelope_json -> 'metrics' -> 'counts' ->> 'completion_tokens')::bigint + ) FILTER ( + WHERE e.envelope_json -> 'payload' ->> 'legacy_kind' = 'model_call' + ), 0) AS completion_tokens + FROM research_event e + WHERE e.agent_run_id = :run_id + """ + ), + {"run_id": str(run_id)}, + ).one() + if savepoint is not None: + savepoint.commit() + except Exception as error: # noqa: BLE001 - finalization must never fail on totals + if savepoint is not None: + savepoint.rollback() + logging.warning(f"[Agent/lifecycle] canonical totals unavailable for {task_id}: {error}") + return None + if not row.steps: + return None + return int(row.steps), int(row.prompt_tokens or 0), int(row.completion_tokens or 0) + + +def finalize_agent_task( + db: Session, task_id: uuid.UUID, status: str = "done" +) -> dict: + """Aggregate a task's events into task-level totals and mark it terminal. + + Safe to call multiple times: totals are recomputed from the same events, so + re-running is idempotent regardless of the task's current status. That + matters because the plugin's close-on-startup call and session deactivation + can both fire for the same task. + """ + events = crud.get_agent_events_by_task(db, task_id) + model_call_events = [e for e in events if e.event_type == "model_call"] + + total_prompt = 0 + total_completion = 0 + agent_profile: Optional[str] = None + tool_names: Optional[list[str]] = None + for event in model_call_events: + total_prompt += event.prompt_tokens or 0 + total_completion += event.completion_tokens or 0 + if agent_profile is None and event.agent_profile: + agent_profile = event.agent_profile + if tool_names is None and event.tool_names_requested: + tool_names = list(event.tool_names_requested) + + last_event = max(events, key=lambda e: e.event_index) if events else None + total_steps = len(events) + if not events: + # A research-bound task's observations live in the canonical store + # (ISSUE-07: never dual-written to agent_event); read the same relay + # facts the dashboards read so the task row is not finalized empty. + canonical = _canonical_totals(db, task_id) + if canonical is not None: + total_steps, total_prompt, total_completion = canonical + crud.update_agent_task_status( + db, + task_id=task_id, + status=status, + completed_at=datetime.now(), + total_steps=total_steps, + input_tokens=total_prompt or None, + output_tokens=total_completion or None, + latest_event_id=last_event.event_id if last_event else None, + ) + # Record the tools the agent actually had available, which for third-party + # runtimes is only knowable after observing a call — the profile's + # tools_json is our *request*, this is the observed reality. + if tool_names is not None: + crud.update_agent_task_tools(db, task_id=task_id, tools=tool_names) + + summary = { + "status": status, + "total_steps": total_steps, + "prompt_tokens": total_prompt, + "completion_tokens": total_completion, + "agent_profile": agent_profile, + "tools": tool_names, + } + logging.info(f"[Agent/lifecycle] task {task_id} finalized — {summary}") + return summary diff --git a/src/agents/normalize.py b/src/agents/normalize.py new file mode 100644 index 0000000..eb83cab --- /dev/null +++ b/src/agents/normalize.py @@ -0,0 +1,624 @@ +"""Wire-format normalization for the agent inference relay. + +Third-party agents don't speak one dialect. This module holds everything needed +to turn what they actually send into something a generic OpenAI-compatible +upstream will accept, and to read telemetry back out of the response: + +* **Chat Completions** (``messages``) — what Goose and the built-in + ``code4me2-agent`` runtime send. Needs JSON-Schema sanitisation, because + Goose's Rust-generated tool schemas carry keywords (``$ref``, ``uint32`` + formats, union ``type`` arrays) that most providers reject outright. +* **Responses API** (``input``) — what Codex sends. OpenAI-proprietary and *not* + chat-completions-shaped, so per merge decision 6 it stays a special-cased + path rather than the default: when the target isn't OpenAI itself, the + proprietary fields have to be stripped or rewritten. + +Also here: extracting token counts / finish reason / response text from both +streaming (SSE) and non-streaming replies, and reconstructing the tool calls an +agent executed locally between two inference requests. +""" + +from __future__ import annotations + +import json +import logging +import re +from typing import Optional + +_SNIPPET_LEN = 300 + + +def snippet(text: str) -> str: + """Collapse text to a single short line for logs and content columns.""" + text = text.strip().replace("\n", " ") + return text[:_SNIPPET_LEN] + "…" if len(text) > _SNIPPET_LEN else text + + +# ── Goose-specific prompt cruft ─────────────────────────────────────────────── + +# Goose prepends an ... block (current time, working +# directory, todo notes) to every user message it sends — strip it so telemetry +# reflects the user's actual request, not this injected context. +_INFO_MSG_RE = re.compile(r".*?", re.DOTALL | re.IGNORECASE) + +# Goose also sends separate "generate a chat title" requests, which wrap the +# real user message(s) like: +# ---BEGIN USER MESSAGES--- +# +# ---END USER MESSAGES--- +# Generate a short title for the above messages. +# Unwrap these so telemetry captures the underlying request, not the title prompt. +_TITLE_GEN_RE = re.compile( + r"---BEGIN USER MESSAGES---(.*?)---END USER MESSAGES---", re.DOTALL +) + + +def strip_info_msg(text: str) -> str: + text = _INFO_MSG_RE.sub("", text) + title_match = _TITLE_GEN_RE.search(text) + if title_match: + text = title_match.group(1) + return text.strip() + + +# Goose fires small auxiliary requests alongside the main conversation — e.g. a +# "summarize this tool call" request after each tool execution, and the +# chat-title request matched by _TITLE_GEN_RE above. These have their own tiny, +# unrelated message arrays and must not be mistaken for (or mask) a real new +# chat session. +_META_SYSTEM_SUBSTRINGS = ("summarize this tool call",) + + +def is_meta_request(current_messages: list) -> bool: + if not current_messages: + return False + for msg in current_messages: + if not isinstance(msg, dict): + continue + content = msg.get("content", "") + if isinstance(content, list): + content = " ".join( + part.get("text", "") for part in content if isinstance(part, dict) + ) + if not isinstance(content, str): + continue + content_lower = content.lower() + if any(p in content_lower for p in _META_SYSTEM_SUBSTRINGS): + return True + if "---begin user messages---" in content_lower: + return True + return False + + +def first_user_message_text( + messages: list, is_responses_api: bool = False +) -> Optional[str]: + """Find the first user-role message's text — used as the task description on + the first inference call of a task, when none was supplied by the plugin.""" + for m in messages: + if not isinstance(m, dict): + continue + if is_responses_api: + if m.get("type") != "message" or m.get("role") != "user": + continue + content = m.get("content", "") + text = ( + " ".join( + c.get("text", "") + for c in content + if isinstance(c, dict) and c.get("type") == "input_text" + ) + if isinstance(content, list) + else str(content) + ) + else: + if m.get("role") != "user": + continue + content = m.get("content", "") + text = content if isinstance(content, str) else "" + text = strip_info_msg(text) + if text: + return text + return None + + +def extract_tool_executions( + messages: list, is_responses_api: bool = False +) -> list[dict]: + """Pair tool-call requests with their results from newly-appended messages. + + Third-party agents execute tools locally and append both the call (assistant + ``tool_calls``) and its result (a ``tool``-role message, matched by + ``tool_call_id``) to the conversation before the next inference request: + + {"role": "assistant", "tool_calls": [{"id": "fc_...", + "function": {"name": "tree", "arguments": "{\\"path\\": \\".\\"}"}}]} + {"role": "tool", "tool_call_id": "fc_...", "content": "src/ [85K]\\n..."} + + That's what lets the proxy record tool_call events without needing a + reporting hook from inside the agent. The Responses API + (``function_call``/``function_call_output``) branch follows the same + item-list shape. Calls without a matching result yet are reported with a + None result. + """ + pending: dict[str, dict] = {} + completed: list[dict] = [] + + if is_responses_api: + for item in messages: + if not isinstance(item, dict): + continue + if item.get("type") == "function_call": + call_id = item.get("call_id") + if call_id: + pending[call_id] = { + "name": item.get("name"), + "arguments": item.get("arguments"), + } + elif item.get("type") == "function_call_output": + call = pending.pop(item.get("call_id"), None) + output = item.get("output") + if isinstance(output, list): + output = " ".join( + c.get("text", "") for c in output if isinstance(c, dict) + ) + completed.append( + { + "name": call.get("name") if call else None, + "arguments": call.get("arguments") if call else None, + "result": output, + } + ) + else: + for m in messages: + if not isinstance(m, dict): + continue + if m.get("role") == "assistant" and isinstance(m.get("tool_calls"), list): + for tc in m["tool_calls"]: + if not isinstance(tc, dict): + continue + call_id = tc.get("id") + fn = tc.get("function") or {} + if call_id: + pending[call_id] = { + "name": fn.get("name"), + "arguments": fn.get("arguments"), + } + elif m.get("role") == "tool": + call = pending.pop(m.get("tool_call_id"), None) + content = m.get("content") + if isinstance(content, list): + content = " ".join( + c.get("text", "") for c in content if isinstance(c, dict) + ) + completed.append( + { + "name": call.get("name") if call else None, + "arguments": call.get("arguments") if call else None, + "result": content, + } + ) + + # Calls still awaiting a result (e.g. tool failed before producing output). + for call in pending.values(): + completed.append( + { + "name": call.get("name"), + "arguments": call.get("arguments"), + "result": None, + } + ) + + return completed + + +# ── JSON Schema sanitisation ───────────────────────────────────────────────── +# +# Fields that OpenAI-compatible tool-calling APIs do not support. +# +# References: +# OpenAI function calling: https://platform.openai.com/docs/guides/function-calling +# OpenAI structured outputs: https://platform.openai.com/docs/guides/structured-outputs#supported-schemas +# Groq tool use: https://console.groq.com/docs/tool-use +# +# Supported subset (keep these): +# type (single string), description, properties, required, enum, items, +# anyOf, minimum/maximum, minLength/maxLength, minItems/maxItems, const, +# default, pattern +_STRIP_KEYS = frozenset( + { + # JSON Schema meta / reference keywords — not part of the OpenAI subset + # https://json-schema.org/understanding-json-schema/structuring + "$schema", + "$ref", + "$defs", + "$id", + "$anchor", + # "title" is valid JSON Schema but several providers reject it at the + # parameter level + "title", + # Explicitly unsupported by OpenAI structured outputs, and a common + # cause of provider-side failed_generation errors: + # https://platform.openai.com/docs/guides/structured-outputs#some-type-specific-keywords-are-not-yet-supported + "additionalProperties", + "unevaluatedProperties", + "patternProperties", + "propertyNames", + "minProperties", + "maxProperties", + # Conditional composition keywords — not in the supported subset + "if", + "then", + "else", + "not", + # Deprecated OpenAI nullable flag; use anyOf: [{type: T}, {type: "null"}] + # https://platform.openai.com/docs/guides/structured-outputs#all-fields-must-be-required + "nullable", + } +) + +# Non-standard numeric format strings providers reject (Goose's Rust-generated +# schemas emit these). The OpenAI spec only recognises "date-time", "email", +# "uri", etc. — not Rust integer widths. +_UNSUPPORTED_FORMATS = frozenset( + {"uint8", "uint16", "uint32", "uint64", "int32", "int64"} +) + + +def _inline_refs(params: dict, defs: dict) -> None: + """Resolve $ref pointers using the $defs map, inlining the definition in-place. + + Only handles local fragment refs of the form "#/$defs/Name". Unknown refs are + left as-is and will be stripped by ``sanitize_schema`` afterwards. + """ + if "$ref" in params: + ref = params.pop("$ref") + if isinstance(ref, str) and ref.startswith("#/$defs/"): + name = ref[len("#/$defs/") :] + resolved = defs.get(name) + if isinstance(resolved, dict): + params.update({k: v for k, v in resolved.items() if k not in params}) + + for child in params.get("properties", {}).values(): + if isinstance(child, dict): + _inline_refs(child, defs) + + for schema_list_key in ("anyOf", "oneOf", "allOf"): + for child in params.get(schema_list_key, []): + if isinstance(child, dict): + _inline_refs(child, defs) + + items = params.get("items") + if isinstance(items, dict): + _inline_refs(items, defs) + + +def sanitize_schema(params: dict) -> None: + """Resolve $ref/$defs, then recursively strip JSON Schema fields providers reject. + + Mutates ``params`` in place. Supported subset: type (string), description, + properties, required, enum, items, anyOf, minimum/maximum, + minLength/maxLength, minItems/maxItems, const, default, pattern. + """ + # Inline $ref references before stripping $defs so no information is lost — + # e.g. Goose's extensionmanager__manage_extensions.action is a $ref to an + # enum in $defs, and dropping it unresolved would lose the allowed values. + defs = params.get("$defs", {}) + if defs: + _inline_refs(params, defs) + + for key in _STRIP_KEYS: + params.pop(key, None) + + fmt = params.get("format") + if isinstance(fmt, str) and fmt in _UNSUPPORTED_FORMATS: + params.pop("format") + + if isinstance(params.get("type"), list): + types = params["type"] + non_null = [t for t in types if t != "null"] + if len(non_null) == 1: + params["type"] = non_null[0] + if params.get("default") is None and "default" in params: + params.pop("default") + + for child in params.get("properties", {}).values(): + if isinstance(child, dict): + sanitize_schema(child) + + items = params.get("items") + if isinstance(items, dict): + sanitize_schema(items) + + for schema_list_key in ("anyOf", "oneOf", "allOf"): + for child in params.get(schema_list_key, []): + if isinstance(child, dict): + sanitize_schema(child) + + +# ── Responses API (Codex) normalization ────────────────────────────────────── + +# Top-level Responses API fields that are OpenAI-proprietary and rejected by +# other providers. Codex always sends these; strip them before forwarding to a +# non-OpenAI upstream. +_RESPONSES_API_STRIP_KEYS = frozenset( + { + "reasoning", # o-series reasoning effort config + "include", # e.g. ["reasoning.encrypted_content"] + "text", # verbosity / format config + "prompt_cache_key", # OpenAI prompt caching + "store", # OpenAI storage flag + "client_metadata", # Codex internal telemetry + "service_tier", # OpenAI tier routing (e.g. "priority") + "parallel_tool_calls", # Chat Completions concept — not in the Responses spec + # Note: `instructions` is handled separately below — it's injected as a + # system message into the input array rather than simply dropped. + } +) + +_VALID_REASONING_EFFORTS = frozenset({"low", "medium", "high"}) + + +def normalize_responses_api_body(body: dict) -> None: + """Rewrite a Codex Responses-API body so a generic provider will accept it. + + Only called when the upstream is *not* OpenAI (see + ``reconcile_openai_passthrough`` for that case). Mutates ``body`` in place: + + * drops OpenAI-proprietary top-level fields; + * moves ``instructions`` into the input array as a system message, since + it's an OpenAI-only top-level field but the content still matters; + * rewrites the OpenAI-only ``developer`` role to ``system``; + * coerces every tool to ``type="function"``. Codex also sends + ``type="custom"`` freeform-grammar tools (apply_patch, exec_command, …) + and hosted tools (web_search, …), which generic providers don't + implement — exposing them as plain functions with a single ``input`` + string keeps them callable, and Codex still executes the result locally. + """ + for key in _RESPONSES_API_STRIP_KEYS: + body.pop(key, None) + + instructions = body.pop("instructions", None) + if instructions: + input_items = body.get("input", []) + if isinstance(input_items, list): + input_items.insert( + 0, + { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": instructions}], + }, + ) + body["input"] = input_items + + for item in body.get("input", []) if isinstance(body.get("input"), list) else []: + if isinstance(item, dict) and item.get("role") == "developer": + item["role"] = "system" + + raw_tools = body.get("tools") + if isinstance(raw_tools, list): + sanitized = [] + for t in raw_tools: + if not isinstance(t, dict) or not t.get("name"): + continue # tools without a name are unusable — drop silently + if t.get("type") == "function": + params = t.get("parameters", {}) + sanitize_schema(params) + sanitized.append( + { + "type": "function", + "name": t["name"], + "description": t.get("description", ""), + "parameters": params, + } + ) + else: + sanitized.append( + { + "type": "function", + "name": t["name"], + "description": t.get("description", ""), + "parameters": { + "type": "object", + "properties": {"input": {"type": "string"}}, + "required": ["input"], + }, + } + ) + body["tools"] = sanitized + body.pop("tool_choice", None) + + +def reconcile_openai_passthrough(body: dict, model: str) -> None: + """Fix up a Responses-API body that *is* going to OpenAI, for a pinned model. + + When the server overrides Codex's requested model, parameters Codex chose + for its original model can become invalid for ours, 400-ing an otherwise + fine request. Reconcile the two known cases: + + * ``gpt-5.1-codex-mini`` only accepts ``text.verbosity == "medium"``, + while Codex defaults to ``"low"`` for other models; + * codex-mini only accepts low/medium/high reasoning effort, so Codex's + ``xhigh`` has to be clamped. + + ``AGENT_REASONING_EFFORT`` is an explicit operator override (e.g. to cap + cost) and takes precedence over the clamp. + """ + import os + + if "codex-mini" in model: + text_cfg = body.get("text") + if isinstance(text_cfg, dict) and text_cfg.get("verbosity") not in ( + None, + "medium", + ): + logging.info( + f"[Agent/normalize] coercing text.verbosity=" + f"{text_cfg.get('verbosity')!r} → 'medium' for {model!r}" + ) + text_cfg["verbosity"] = "medium" + + reasoning_cfg = body.get("reasoning") + if not isinstance(reasoning_cfg, dict): + return + + effort_override = os.getenv("AGENT_REASONING_EFFORT", "").strip() + if effort_override: + if reasoning_cfg.get("effort") != effort_override: + logging.info( + f"[Agent/normalize] reasoning.effort override via env — " + f"{reasoning_cfg.get('effort')!r} → {effort_override!r}" + ) + reasoning_cfg["effort"] = effort_override + return + + if "codex-mini" in model: + effort = reasoning_cfg.get("effort") + if isinstance(effort, str) and effort not in _VALID_REASONING_EFFORTS: + logging.info( + f"[Agent/normalize] coercing reasoning.effort={effort!r} → 'high' " + f"for {model!r}" + ) + reasoning_cfg["effort"] = "high" + + +# ── Response parsing ───────────────────────────────────────────────────────── + +_ParsedResponse = tuple[ + Optional[int], Optional[int], Optional[int], Optional[str], Optional[str] +] + + +def parse_stream(raw_sse: str) -> _ParsedResponse: + """Extract (prompt_tokens, completion_tokens, total_tokens, finish_reason, + response_text) from a Chat Completions SSE stream. + + Token counts only appear when the request set + ``stream_options.include_usage``; otherwise they come back None. + """ + content_parts: list[str] = [] + finish_reason = None + prompt_tokens = completion_tokens = total_tokens = None + + for line in raw_sse.splitlines(): + if not line.startswith("data:"): + continue + data = line[5:].strip() + if data == "[DONE]": + continue + try: + chunk = json.loads(data) + except json.JSONDecodeError: + continue + + # Usage chunk. OpenAI (stream_options.include_usage) sends it with an + # empty choices array; OpenRouter attaches it to the final chunk next + # to a choice with an empty delta. Either way: the last chunk whose + # ``usage`` is an object wins, and its choices are still processed. + usage = chunk.get("usage") + if isinstance(usage, dict): + prompt_tokens = usage.get("prompt_tokens") + completion_tokens = usage.get("completion_tokens") + total_tokens = usage.get("total_tokens") + + for choice in chunk.get("choices") or []: + delta_content = choice.get("delta", {}).get("content") + if delta_content: + content_parts.append(delta_content) + if choice.get("finish_reason"): + finish_reason = choice["finish_reason"] + + return ( + prompt_tokens, + completion_tokens, + total_tokens, + finish_reason, + "".join(content_parts) or None, + ) + + +def parse_responses_api_stream(raw_sse: str) -> _ParsedResponse: + """Same extraction for a Responses API SSE stream. + + The Responses API uses named events (``response.output_text.delta``, + ``response.completed``) rather than Chat Completions' ``choices[].delta`` + shape, so it needs its own parser — without this, every streaming Codex + call records null tokens and no response text. + """ + content_parts: list[str] = [] + finish_reason = None + prompt_tokens = completion_tokens = total_tokens = None + + for line in raw_sse.splitlines(): + if not line.startswith("data:"): + continue + data = line[5:].strip() + if not data or data == "[DONE]": + continue + try: + chunk = json.loads(data) + except json.JSONDecodeError: + continue + if not isinstance(chunk, dict): + continue + + event_type = chunk.get("type") or "" + if event_type.endswith("output_text.delta"): + delta = chunk.get("delta") + if isinstance(delta, str): + content_parts.append(delta) + elif event_type in ("response.completed", "response.incomplete"): + response = chunk.get("response") or {} + usage = response.get("usage") or {} + # The Responses API names these input/output rather than + # prompt/completion. + prompt_tokens = usage.get("input_tokens", usage.get("prompt_tokens")) + completion_tokens = usage.get( + "output_tokens", usage.get("completion_tokens") + ) + total_tokens = usage.get("total_tokens") + finish_reason = response.get("status") or ( + "completed" if event_type == "response.completed" else "incomplete" + ) + + return ( + prompt_tokens, + completion_tokens, + total_tokens, + finish_reason, + "".join(content_parts) or None, + ) + + +def extract_from_response(resp_json: dict) -> _ParsedResponse: + """Extract the same tuple from a non-streaming Chat Completions response.""" + usage = resp_json.get("usage", {}) or {} + choice = (resp_json.get("choices") or [{}])[0] + content = (choice.get("message", {}) or {}).get("content") or "" + return ( + usage.get("prompt_tokens"), + usage.get("completion_tokens"), + usage.get("total_tokens"), + choice.get("finish_reason"), + content or None, + ) + + +def extract_from_responses_api(resp_json: dict) -> _ParsedResponse: + """Extract the same tuple from a non-streaming Responses API response.""" + usage = resp_json.get("usage", {}) or {} + text_parts: list[str] = [] + for item in resp_json.get("output", []) or []: + if not isinstance(item, dict): + continue + for part in item.get("content", []) or []: + if isinstance(part, dict) and isinstance(part.get("text"), str): + text_parts.append(part["text"]) + return ( + usage.get("input_tokens", usage.get("prompt_tokens")), + usage.get("output_tokens", usage.get("completion_tokens")), + usage.get("total_tokens"), + resp_json.get("status"), + "".join(text_parts) or None, + ) diff --git a/src/agents/provider.py b/src/agents/provider.py new file mode 100644 index 0000000..5cda841 --- /dev/null +++ b/src/agents/provider.py @@ -0,0 +1,279 @@ +"""Upstream LLM provider resolution for the agent inference relay. + +Provider endpoints and secrets are administrator-managed facts: a +``provider_connection`` row owns the OpenAI-compatible ``base_url`` and the +*name* of the deployment secret (``secret_ref``) that holds the key. A profile +selects a connection and a model. Secrets are resolved from the environment only +at actual request time and are never stored, logged, snapshotted or returned. + +There is deliberately no fallback: a missing/inactive connection, a missing +secret, or a model not allowed by the connection is a readiness failure, never +an accidental use of a server default or another owner's connection. + +Codex's proprietary Responses API is not chat-completions-shaped, so it stays a +special-cased normalization path keyed on ``framework_version == "codex"``. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +from dataclasses import dataclass +from typing import Optional +from urllib.parse import urlparse + +__all__ = [ + "ProviderReadinessError", + "ResolvedConnection", + "Upstream", + "connection_model_allowed", + "connection_view", + "resolve_api_key", + "resolve_task_connection", + "resolve_upstream", +] + + +@dataclass(frozen=True) +class ResolvedConnection: + """A session-independent snapshot of a provider connection. + + Taking primitives out of the ORM row means the resolved connection stays + usable after the request session closes, without holding a lazy relationship. + """ + + connection_id: object + label: str + base_url: str + secret_ref: str + models_json: str + is_active: bool + + +def connection_view(connection) -> ResolvedConnection: + return ResolvedConnection( + connection_id=connection.connection_id, + label=connection.label, + base_url=connection.base_url, + secret_ref=connection.secret_ref, + models_json=connection.models_json, + is_active=bool(connection.is_active), + ) + + +def resolve_task_connection( + db, + profile, + owner_user_id, + *, + owner_is_admin: bool = False, +) -> ResolvedConnection: + """Resolve and authorize the connection a task's profile selects. + + Re-checked at every inference request: the connection must exist and be + active. Authorization is role-based (the funding owner must be an enabled + researcher or an administrator); readiness is connection state + (active + allowed model + present secret), not a per-owner grant row. + ``owner_user_id`` is retained so callers keep passing the funding owner + explicitly, but no per-owner grant is consulted. + """ + from database import crud # local import keeps provider.py dependency-light + + connection_id = getattr(profile, "connection_id", None) + if connection_id is None: + raise ProviderReadinessError( + "CONNECTION_MISSING", + "the task's profile does not reference a provider connection", + ) + connection = crud.get_provider_connection(db, connection_id) + if connection is None: + raise ProviderReadinessError( + "CONNECTION_MISSING", + "the task's provider connection no longer exists", + ) + if not owner_is_admin: + if owner_user_id is None or not crud.provider_connection_is_available( + db, connection.connection_id, owner_user_id + ): + raise ProviderReadinessError( + "CONNECTION_NOT_READY", + f"provider connection {connection.label!r} is not ready", + ) + return connection_view(connection) + + +def funding_owner_for_task(db, task): + """Return ``(funding_owner_user_id, owner_is_admin)`` for connection auth. + + The grant belongs to the researcher who owns the study that funds a task, not + the participant who ran it. The frozen ``task.funding_owner_user_id`` is + authoritative; ``Study.created_by`` is a fallback for tasks created before + the field was populated. A participant's own admin flag never applies. + """ + from database import crud + from database.db_schemas import Study as StudyRow + + funding_owner_user_id = getattr(task, "funding_owner_user_id", None) + if funding_owner_user_id is None and getattr(task, "study_id", None) is not None: + study = db.get(StudyRow, task.study_id) + if study is not None: + funding_owner_user_id = getattr(study, "created_by", None) + return _funding_owner_admin(db, funding_owner_user_id) + + +def funding_owner_for_profile(db, profile): + """Return ``(funding_owner_user_id, owner_is_admin)`` for a frozen profile.""" + return _funding_owner_admin( + db, getattr(profile, "funding_owner_user_id", None) + ) + + +def _funding_owner_admin(db, funding_owner_user_id): + from database import crud + + owner_is_admin = False + if funding_owner_user_id is not None: + owner = crud.get_user_by_id(db, funding_owner_user_id) + owner_is_admin = bool(owner is not None and owner.is_admin) + return funding_owner_user_id, owner_is_admin + + +class ProviderReadinessError(RuntimeError): + """A typed, actionable failure to resolve a usable provider connection. + + The message is safe to return to a caller: it names the reason and the + connection label, never a secret value or the raw URL secret. + """ + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +@dataclass(frozen=True) +class Upstream: + """A resolved OpenAI-compatible upstream target.""" + + base_url: str + api_key: str + model: str + # Which agent runtime this task belongs to: code4me2-agent | goose | codex. + framework_version: Optional[str] = None + + @property + def is_responses_api(self) -> bool: + """Whether this runtime speaks OpenAI's Responses API rather than + chat-completions. Only Codex does.""" + return (self.framework_version or "").strip().lower() == "codex" + + def endpoint(self, *, responses_api: bool) -> str: + """Full URL for the request, given the wire shape actually observed.""" + suffix = "responses" if responses_api else "chat/completions" + return f"{self.base_url.rstrip('/')}/{suffix}" + + +# OpenCode (Zen and Go) rejects a request without ``x-opencode-session`` +# (400 MissingSessionID) and asks clients to send their own user agent. +OPENCODE_USER_AGENT = "code4me-research/1.0" +_OPENCODE_SESSION_DOMAIN = b"code4me-opencode-session\x00" + + +def is_opencode_upstream(base_url: Optional[str]) -> bool: + """Whether ``base_url`` points at OpenCode's gateway (opencode.ai or a subdomain).""" + host = (urlparse(base_url or "").hostname or "").lower() + return host == "opencode.ai" or host.endswith(".opencode.ai") + + +def upstream_request_headers(upstream: "Upstream", *, session_key: Optional[str]) -> dict[str, str]: + """Headers for one provider call. + + Every provider receives only the bearer key, exactly as before. OpenCode + additionally gets our user agent and a session id that stays stable for one + agent session (``session_key``) so it can route and cache the conversation. + The id is a one-way hash, so no internal id or token leaves the backend. + """ + headers = {"Authorization": f"Bearer {upstream.api_key}"} + if is_opencode_upstream(upstream.base_url): + headers["User-Agent"] = OPENCODE_USER_AGENT + if session_key: + digest = hashlib.sha256(_OPENCODE_SESSION_DOMAIN + session_key.encode("utf-8")).hexdigest() + headers["x-opencode-session"] = f"c4m-{digest[:32]}" + return headers + +def connection_model_allowed(connection, model: str) -> bool: + """Whether ``model`` is in the connection's allowed model list.""" + try: + models = json.loads(getattr(connection, "models_json", None) or "[]") + except (json.JSONDecodeError, TypeError): + models = [] + return str(model) in {str(item) for item in models} + + +def resolve_api_key(secret_ref: Optional[str]) -> str: + """Read a provider secret from the environment by its variable *name*. + + Raises :class:`ProviderReadinessError` when the reference is unset or the + environment variable is empty: there is no unauthenticated fallback, so a + missing secret is a clear readiness failure rather than a silent 401. + """ + env_name = (secret_ref or "").strip() + if not env_name: + raise ProviderReadinessError( + "SECRET_REF_MISSING", + "the provider connection has no secret reference configured", + ) + key = os.getenv(env_name, "").strip() + if not key: + raise ProviderReadinessError( + "SECRET_MISSING", + f"provider secret ${env_name} is not present in the environment", + ) + return key + + +def resolve_upstream( + *, + model: str, + connection, + framework_version: Optional[str] = None, +) -> Upstream: + """Build the upstream target from the task's frozen connection identity. + + The connection is re-validated here, at request time, so a revoked/disabled + connection blocks both new and previously created tasks. The secret value is + read from the environment and never logged. + """ + if connection is None: + raise ProviderReadinessError( + "CONNECTION_MISSING", + "the task's profile does not reference a provider connection", + ) + if not getattr(connection, "is_active", False): + raise ProviderReadinessError( + "CONNECTION_INACTIVE", + f"provider connection {getattr(connection, 'label', '?')!r} is not active", + ) + if not connection_model_allowed(connection, model): + raise ProviderReadinessError( + "MODEL_NOT_ALLOWED", + f"model {model!r} is not allowed by connection " + f"{getattr(connection, 'label', '?')!r}", + ) + + resolved = Upstream( + base_url=str(connection.base_url).strip(), + api_key=resolve_api_key(getattr(connection, "secret_ref", None)), + model=model, + framework_version=framework_version, + ) + logging.info( + "[Agent/provider] connection=%r model=%r runtime=%s authenticated=%s", + getattr(connection, "label", None), + resolved.model, + resolved.framework_version or "unknown", + bool(resolved.api_key), + ) + return resolved diff --git a/src/agents/registry.py b/src/agents/registry.py new file mode 100644 index 0000000..f1f51d1 --- /dev/null +++ b/src/agents/registry.py @@ -0,0 +1,245 @@ +"""Server-authoritative agent assignment for research tasks. + +There is exactly one assignment authority: the research ``study_assignment`` row +allocated by :mod:`research.runtime.assignment`. Bootstrap and task creation both +resolve that same persisted row, so a participant always gets one sticky agent +profile per enrollment. + +A research run requires a real participant mapping, an **active** enrollment, an +open research study window, and a selected profile snapshot. When any of those +is missing this returns ``None``; mutable profile templates are never used as a +runtime fallback. +""" + +from __future__ import annotations + +import logging +import json +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Optional + +from sqlalchemy.orm import Session + +from database.db_schemas import Study as StudyRow +from research.participants import identity as identity_store +from research.participants.enums import EnrollmentStatus +from research.runtime.assignment import store as assignment_store +from research.runtime.assignment.enums import AllocationOutcome +from research.runtime.assignment.hashing import assignment_strategy +from research.runtime.assignment.models import StudyProfileSelection +from research.runtime.assignment.service import allocate +from research.study.protocol import store as protocol_store + + +@dataclass(frozen=True) +class FrozenAgentConfig: + """The immutable execution config of one assigned condition. + + Built only from the revision's frozen ``resolved_distribution`` (and its + ``agent_config``). It is a plain value object, never a mutable ORM + ``AgentProfile`` row, so editing a profile template cannot change a + published study's tasks or a bootstrap manifest. + """ + + profile_id: uuid.UUID + name: str + model: str + framework_version: str + tools_json: str + approval_policy: str + max_steps: int + temperature: Optional[float] = None + max_context_tokens: Optional[int] = None + connection_id: Optional[uuid.UUID] = None + connection_label: Optional[str] = None + release_id: Optional[str] = None + distribution_mode: str = "PACKAGED" + artifact_digest: Optional[str] = None + agent_command: Optional[str] = None + agent_command_args: list[str] = field(default_factory=list) + # The researcher whose connection grant funds execution (study owner). + funding_owner_user_id: Optional[uuid.UUID] = None + # Researcher-authored system prompt frozen with the study selection; None = + # no prompt (snapshots taken before the field existed never carry one). + system_prompt: Optional[str] = None + # Built-in runtime command/harness settings frozen with the selection + # (decision D-01); None = not set (older snapshots never carry them). Copied + # verbatim: the agent-config and run-policy builders validate them and fail + # closed on a malformed snapshot instead of silently dropping a setting. + commands_denylist: Optional[list[str]] = None + command_timeout_seconds: Optional[int] = None + harness_options: Optional[dict[str, Any]] = None + # Legacy: the allowlist a snapshot frozen before ``commands_denylist`` + # carries. It only reaches runtime releases 0.0.6 and earlier. + commands_allowlist: Optional[list[str]] = None + + +@dataclass(frozen=True) +class AgentAssignmentResolution: + """The frozen profile and immutable experiment arm for one task.""" + + profile: FrozenAgentConfig + study_id: Optional[uuid.UUID] = None + assignment_id: Optional[uuid.UUID] = None + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _active_enrollment(db: Session, participant_id: uuid.UUID): + """Return the participant's active enrollment row, or ``None``. + + One active enrollment per account is enforced by a partial unique index, so + there is at most one candidate. + """ + for row in identity_store.list_enrollments(db, participant_id): + if row.status == EnrollmentStatus.ACTIVE.value: + return row + return None + + +def _frozen_config_from_snapshot( + profile_snapshot: dict, funding_owner_user_id: Optional[uuid.UUID] +) -> Optional[FrozenAgentConfig]: + """Build runtime config only from the assignment's immutable snapshot.""" + profile_id = profile_snapshot.get("profile_id") + if not profile_id: + return None + tools_json = profile_snapshot.get("tools_json", "[]") + if not isinstance(tools_json, str): + tools_json = json.dumps(tools_json, separators=(",", ":")) + system_prompt = profile_snapshot.get("system_prompt") + return FrozenAgentConfig( + profile_id=uuid.UUID(str(profile_id)), + name=profile_snapshot.get("name", "assigned-profile"), + model=profile_snapshot.get("model", ""), + framework_version=profile_snapshot.get("framework_version", "code4me2-agent"), + tools_json=tools_json, + approval_policy=profile_snapshot.get("approval_policy", "per_step"), + max_steps=profile_snapshot.get("max_steps", 1), + temperature=profile_snapshot.get("temperature"), + max_context_tokens=profile_snapshot.get("max_context_tokens"), + connection_id=( + uuid.UUID(str(profile_snapshot["connection_id"])) + if profile_snapshot.get("connection_id") + else None + ), + release_id=profile_snapshot.get("release_id"), + funding_owner_user_id=funding_owner_user_id, + system_prompt=system_prompt if isinstance(system_prompt, str) else None, + commands_denylist=profile_snapshot.get("commands_denylist"), + command_timeout_seconds=profile_snapshot.get("command_timeout_seconds"), + harness_options=profile_snapshot.get("harness_options"), + commands_allowlist=profile_snapshot.get("commands_allowlist"), + ) + + +def resolve_assignment_context( + db: Session, user_id: uuid.UUID +) -> Optional[AgentAssignmentResolution]: + """Resolve the frozen profile/assignment for a new research task. + + Steps: participant mapping → active enrollment → open research study → + sticky ``study_assignment`` → frozen profile snapshot. Any missing step is + a refusal (``None``). + """ + participant_row = identity_store.get_participant_by_account(db, user_id) + if participant_row is None: + logging.info( + "[Agent/registry] account %s has no research participant mapping; " + "refusing assignment", + str(user_id)[:8], + ) + return None + + enrollment_row = _active_enrollment(db, participant_row.participant_id) + if enrollment_row is None: + logging.info( + "[Agent/registry] participant %s has no active enrollment; refusing " + "assignment", + str(participant_row.participant_id)[:8], + ) + return None + + enrollment = identity_store.row_to_enrollment(enrollment_row) + study = db.get(StudyRow, enrollment.study_id) + now = _now() + if not protocol_store.research_study_is_open(study, now): + logging.info( + "[Agent/registry] study %s is not open for execution; refusing assignment", + enrollment.study_id, + ) + return None + + assignment_row = assignment_store.get_assignment_for_enrollment( + db, enrollment.enrollment_id + ) + if assignment_row is None: + from sqlalchemy import select + from database.research_schemas import StudyAgentProfile + + profile_rows = db.execute( + select(StudyAgentProfile) + .where(StudyAgentProfile.study_id == enrollment.study_id) + .order_by(StudyAgentProfile.selection_order.asc()) + ).scalars().all() + profiles = [ + StudyProfileSelection( + study_id=row.study_id, + agent_profile_id=row.profile_id, + profile_digest=row.profile_digest, + profile_snapshot_json=row.profile_snapshot_json, + selection_order=row.selection_order, + ) + for row in profile_rows + ] + allocation = allocate( + enrollment, + profiles, + existing=None, + now=now, + strategy=assignment_strategy(getattr(study, "research_config_json", None)), + ) + if ( + allocation.outcome + not in (AllocationOutcome.CREATED, AllocationOutcome.EXISTING) + or allocation.assignment is None + ): + logging.warning( + "[Agent/registry] allocation refused: %s", + getattr(allocation.issue, "code", allocation.outcome), + ) + return None + assignment = allocation.assignment + if allocation.created: + # create_assignment returns the winning row on a concurrent first-use + # race, so this context adopts the authoritative sticky profile. + winner_row = assignment_store.create_assignment(db, assignment) + if winner_row is not None and winner_row.assignment_id != assignment.assignment_id: + assignment = assignment_store.row_to_assignment(winner_row) + + if assignment_row is not None: + assignment = assignment_store.row_to_assignment(assignment_row) + profile = _frozen_config_from_snapshot( + assignment.profile_snapshot_json, getattr(study, "created_by", None) + ) + if profile is None: + logging.error( + "[Agent/registry] assignment has no frozen profile snapshot; refusing assignment", + ) + return None + + return AgentAssignmentResolution( + profile=profile, + study_id=enrollment.study_id, + assignment_id=assignment.assignment_id, + ) + + +def resolve_assignment(db: Session, user_id: uuid.UUID) -> Optional[FrozenAgentConfig]: + """Return the frozen profile selected for a new task, or ``None``.""" + resolution = resolve_assignment_context(db, user_id) + return resolution.profile if resolution is not None else None diff --git a/src/agents/telemetry.py b/src/agents/telemetry.py new file mode 100644 index 0000000..7ba98be --- /dev/null +++ b/src/agents/telemetry.py @@ -0,0 +1,129 @@ +"""Telemetry record captured per LLM inference call intercepted by the proxy. + +One ``InferenceRecord`` maps onto one ``agent_event`` row with +``event_type='model_call'``. Every field is a typed column — this is the +"columnar core" half of the merged schema; adapter-specific extras that don't +warrant a column go into ``agent_event.extra_json`` instead. + +Content fields (``first_system_message``, ``last_user_message``, +``response_text``) are only ever populated when the requesting user's +``store_agent_content`` preference resolves True, decided server-side. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Optional + +from pydantic import BaseModel, Field + + +class InferenceRecord(BaseModel): + request_id: str = Field( + description="UUID generated per call. Used as the model_call span_id." + ) + session_id: Optional[str] = Field( + default=None, + description="Code4Me session identifier, resolved server-side from the " + "session cookie. Links all inference calls made within one IDE session.", + ) + task_id: Optional[str] = Field( + default=None, + description="Agent task identifier. Links this call to an AgentTask row " + "so all inference steps of one task are grouped.", + ) + agent_profile: Optional[str] = Field( + default=None, + description="Profile name assigned for this task (e.g. 'default-goose'). " + "Allows per-arm analysis of inference behaviour.", + ) + event_type: str = Field( + default="model_call", + description="Event type for the agent_event table. Always 'model_call' " + "for proxy-captured inference calls.", + ) + timestamp: datetime = Field( + default_factory=lambda: datetime.now(timezone.utc), + description="UTC timestamp when the request was received by the relay.", + ) + model: str = Field( + description="Model actually sent upstream, after the server-side override " + "from the task's snapshotted profile." + ) + streaming: bool = Field( + description="Whether the client requested a streaming response." + ) + message_count: int = Field( + description="Total messages in the conversation context sent to the model. " + "Indicates how deep into a task this call occurred." + ) + role_breakdown: dict[str, int] = Field( + default_factory=dict, + description="Count of messages per role (system, user, assistant, tool). " + "Captures the shape of the context window.", + ) + first_system_message: Optional[str] = Field( + default=None, + description="Text of the system-role message. Content — gated by " + "store_agent_content.", + ) + last_user_message: Optional[str] = Field( + default=None, + description="Text of the most recent user-role message. Content — gated by " + "store_agent_content.", + ) + tools_kept: int = Field( + default=0, + description="Number of tool definitions forwarded to the upstream model.", + ) + tools_stripped: int = Field( + default=0, + description="Number of tool definitions removed before forwarding, either " + "because the profile's allowlist excluded them or the provider rejects " + "their schema. A non-zero value here is how you discover a tool name " + "missing from agents.tools.KNOWN_AGENT_TOOLS.", + ) + tool_names_requested: list[str] = Field( + default_factory=list, + description="Function names of all tools in the client request before " + "filtering. Records what the agent had available regardless of which were " + "forwarded upstream.", + ) + max_tokens: Optional[int] = Field( + default=None, description="max_tokens parameter from the client request." + ) + active_file: Optional[str] = Field( + default=None, + description="Path of the file active in the IDE editor when this call was " + "made, supplied by the plugin via enrichment. Structural metadata — " + "stored regardless of the content-storage preference.", + ) + prompt_tokens: Optional[int] = Field( + default=None, + description="Input token count as reported by the upstream provider. None " + "for streaming calls when the provider returns no usage chunk.", + ) + completion_tokens: Optional[int] = Field( + default=None, description="Output token count as reported by the provider." + ) + total_tokens: Optional[int] = Field( + default=None, description="Total tokens as reported by the provider." + ) + finish_reason: Optional[str] = Field( + default=None, + description="Why the model stopped ('stop', 'tool_calls', 'length', or " + "'stream_aborted' when the relay lost the connection mid-flight). Useful " + "for detecting truncated responses.", + ) + response_text: Optional[str] = Field( + default=None, + description="Text of the model's response. Content — gated by " + "store_agent_content.", + ) + latency_ms: int = Field( + description="End-to-end relay latency in ms (request received → full " + "response returned). For streaming calls, until the last chunk is flushed." + ) + upstream_status: int = Field( + description="HTTP status code returned by the upstream provider." + ) diff --git a/src/agents/tool_call_filter.py b/src/agents/tool_call_filter.py new file mode 100644 index 0000000..8804db4 --- /dev/null +++ b/src/agents/tool_call_filter.py @@ -0,0 +1,385 @@ +"""Withhold model tool calls to tools outside an enforced selection. + +Offering the model only the selected tools is not enough on its own: a model +can still emit a call to a tool it was never offered (its agent's system prompt +may describe it), and an agent such as Goose executes a call to any tool its +extensions provide. Where a tool selection is enforced (the research inference +gateway, for a release that declares it), the response is filtered too, so the +agent never holds a call to a withheld tool. + +Both Chat Completions shapes are handled. A JSON body simply loses its +withheld calls. A streamed response is filtered against the way Goose 1.51 +assembles calls (``goose-provider-types`` ``formats/openai.rs``). Goose reads +only the first choice and collects calls in episodes: + +* a chunk with tool calls starts one: each entry with an ``id`` and a ``name`` + creates a call keyed by its ``index``, else its position (the last entry for + a key wins); other entries are ignored. A ``tool_calls`` or ``length`` + finish ends the episode at once; +* otherwise later chunks continue it until one finishes (any reason) or has no + choices. Their entries need an ``index``: on a known key they append + arguments and a non-empty ``name`` renames the call; an unknown key is + created by an entry with an ``id`` and a ``name``; +* the calls held when the episode ends are run. + +The filter follows the provider's stream by these rules and forwards only what +keeps the agent in step with it. A key the stream creates or renames as a tool +outside the selection (an empty name included) is poisoned for the episode: +that entry and every later one for the key are dropped. So the agent starts an +episode only with an allowed call, and reads each forwarded chunk as the +stream's own: + +* an index-less first-chunk entry behind a dropped one gets its position as + ``index``, the key Goose gives it anyway; +* after every earlier call of an episode was withheld, the agent's first chunk + is a later one: a key repeated within it is dropped, and a finish other than + ``tool_calls`` or ``length`` becomes ``tool_calls``, so both episodes end + together. + +A rename of a forwarded call to a withheld tool is dropped with the rest of its +key; the agent keeps the allowed call. When a call is withheld and the agent +holds none, that chunk carries a short notice (once per episode), and a +``tool_calls`` finish of such an episode becomes ``stop``. Calls in any other +choice are removed. + +The stream is read as Goose reads it (``LinesCodec``): line by line, split on +LF with one trailing CR dropped, each ``data:`` line a chunk of its own, its +value trimmed of Unicode whitespace. What only Goose, or only an SSE reader, +could take as a chunk is dropped (fail closed): a line with any other CR, a +``data:`` line that is not valid JSON or UTF-8, and a chunk in which a choice, a +delta, a call or its ``function`` is not a JSON object (Goose's deserializer +also reads those from arrays). Lines that need no change pass through byte for +byte. +""" + +from __future__ import annotations + +import json +from typing import AbstractSet, Any, Optional + +__all__ = ["WITHHELD_NOTICE", "SseToolCallFilter", "filter_chat_completion_body", "tool_allowed"] + +#: Told to the participant (and the model, in its history) when a turn's only +#: tool calls were withheld, so the turn does not end in silence. +WITHHELD_NOTICE = "[Code4Me study] The model tried to use a tool this study does not allow; it was not run." + +#: What Rust's ``str::trim`` removes (Unicode ``White_Space``); Goose trims ``data:`` values with it. +_TRIMMED = "\t\n\x0b\x0c\r \x85\xa0\u1680" + "".join(map(chr, range(0x2000, 0x200B))) + "\u2028\u2029\u202f\u205f\u3000" + + +def tool_allowed(name: Any, allowlist: AbstractSet[str]) -> bool: + """Whether ``name`` is in ``allowlist`` (``mcp__*`` admits every ``mcp__`` tool).""" + return ( + isinstance(name, str) + and bool(name) + and (name in allowlist or ("mcp__*" in allowlist and name.startswith("mcp__"))) + ) + + +def _function_name(call: Any) -> Optional[str]: + function = call.get("function") if isinstance(call, dict) else None + name = function.get("name") if isinstance(function, dict) else None + return name if isinstance(name, str) else None + + +def filter_chat_completion_body(body: Any, allowlist: AbstractSet[str]) -> list[str]: + """Filter a non-streamed Chat Completions response in place; return the withheld names.""" + withheld: list[str] = [] + if not isinstance(body, dict): + return withheld + for choice in body.get("choices") or []: + message = choice.get("message") if isinstance(choice, dict) else None + if not isinstance(message, dict): + continue + removed_all = False + calls = message.get("tool_calls") + if isinstance(calls, list): + kept = [call for call in calls if tool_allowed(_function_name(call), allowlist)] + withheld.extend(str(_function_name(call)) for call in calls if call not in kept) + if kept: + message["tool_calls"] = kept + else: + message.pop("tool_calls", None) + removed_all = bool(calls) + if choice.get("finish_reason") == "tool_calls": + choice["finish_reason"] = "stop" + legacy = message.get("function_call") + if isinstance(legacy, dict) and not tool_allowed(legacy.get("name"), allowlist): + withheld.append(str(legacy.get("name"))) + message.pop("function_call", None) + removed_all = True + if choice.get("finish_reason") == "function_call": + choice["finish_reason"] = "stop" + if removed_all and not message.get("content"): + message["content"] = WITHHELD_NOTICE + return withheld + + +def _not_json(constant: str) -> Any: + raise ValueError(f"{constant} is not JSON") + + +def _finite(literal: str) -> float: + number = float(literal) + if number in (float("inf"), float("-inf")): + raise ValueError(f"{literal} is out of range") + return number + + +def _readable_shape(event: dict) -> bool: + """Whether every choice, delta, call and ``function`` in the chunk is a JSON object.""" + if "choices" not in event: + return True + choices = event["choices"] + if not isinstance(choices, list): + return False + for choice in choices: + if not isinstance(choice, dict) or not isinstance(choice.get("delta", {}), dict): + return False + calls = choice.get("delta", {}).get("tool_calls") + if calls is not None and not ( + isinstance(calls, list) + and all(isinstance(call, dict) and isinstance(call.get("function"), dict) for call in calls) + ): + return False + return True + + +#: A finish that ends an episode in its first chunk (any finish ends a later one). +_ENDS_AT_ONCE = ("tool_calls", "length") + + +def _index(call: dict) -> Optional[int]: + index = call.get("index") + return index if isinstance(index, int) and not isinstance(index, bool) else None + + +def _creates(call: Any) -> bool: + """Whether Goose creates a call from this entry: ``id`` and ``name`` are present.""" + return isinstance(call, dict) and isinstance(call.get("id"), str) and _function_name(call) is not None + + +class _Episode: + """The first choice's current episode, followed on the provider's stream.""" + + def __init__(self, upstream: bool = False) -> None: + self.upstream = upstream # the provider's stream is inside an episode + self.agent = False # so is the agent: it holds at least one call + self.held: dict[int, str] = {} # the agent's calls: key -> name + self.poisoned: set[int] = set() + self.withheld = False + self.noticed = False + + +class SseToolCallFilter: + """Stateful filter for one streamed Chat Completions response (see the module doc).""" + + def __init__(self, allowlist: AbstractSet[str]) -> None: + self._allowlist = allowlist + self._buffer = b"" + self._episode = _Episode() + self._legacy_allowed: Optional[bool] = None + self.withheld: list[str] = [] + self.dropped = 0 # lines not forwarded because they could not be judged + + def feed(self, chunk: bytes) -> bytes: + """Accept upstream bytes; return the filtered bytes of every complete line.""" + searched = len(self._buffer) # the buffer holds no LF before this chunk + self._buffer += chunk + out: list[bytes] = [] + start = 0 + while (end := self._buffer.find(b"\n", max(start, searched))) >= 0: + out.append(self._line(self._buffer[start : end + 1])) + start = end + 1 + self._buffer = self._buffer[start:] + return b"".join(out) + + def flush(self) -> bytes: + """The filtered remainder once the upstream stream has ended (Goose reads it as a last line).""" + rest, self._buffer = self._buffer, b"" + return self._line(rest) if rest else b"" + + def _line(self, raw: bytes) -> bytes: + body = raw[:-1] if raw.endswith(b"\n") else raw + body = body[:-1] if body.endswith(b"\r") else body + ending = raw[len(body) :] + if b"\r" in body: + self.dropped += 1 # Goose and SSE readers split this line differently + return b"" + if not body.startswith(b"data:"): + return raw + try: + value: Optional[str] = body[5:].decode("utf-8").strip(_TRIMMED) + except UnicodeDecodeError: + value = None + if value in ("", "[DONE]"): + return raw # carries no chunk + try: + event = json.loads(value, parse_constant=_not_json, parse_float=_finite) if value is not None else None + except (ValueError, RecursionError): + event = None + if not isinstance(event, dict) or not _readable_shape(event): + self.dropped += 1 + return b"" + if not self._filter_chunk(event): + return raw + rewritten = json.dumps(event, separators=(",", ":"), ensure_ascii=True, allow_nan=False) + return b"data: " + rewritten.encode("ascii") + ending + + def _filter_chunk(self, event: dict) -> bool: + choices = event.get("choices") + if not isinstance(choices, list): + return False + if not choices: + self._episode = _Episode() # a chunk without choices ends the episode + return False + changed = False + for choice in choices[1:]: + changed |= self._drop_calls(choice) + if isinstance(choices[0], dict): + changed |= self._filter_first_choice(choices[0]) + return changed + + def _refuse(self, key: int, name: str) -> None: + self._episode.poisoned.add(key) + self._episode.withheld = True + if name: + self.withheld.append(name) + + def _first_chunk_calls(self, calls: list) -> list: + episode = self._episode + keys: list[Optional[int]] = [] + final: dict[int, str] = {} + for position, call in enumerate(calls): + key = None + if _creates(call): + index = _index(call) + key = position if index is None else index + final[key] = _function_name(call) + keys.append(key) + for key, name in final.items(): + if tool_allowed(name, self._allowlist): + episode.held[key] = name + else: + self._refuse(key, name) + kept: list = [] + for position, (call, key) in enumerate(zip(calls, keys)): + if key is None or key in episode.poisoned or not tool_allowed(_function_name(call), self._allowlist): + continue + if _index(call) is None and len(kept) != position: + call = {**call, "index": position} + kept.append(call) + return kept + + def _later_chunk_calls(self, calls: list) -> list: + episode = self._episode + agents_first = not episode.agent + created: set[int] = set() + kept: list = [] + for call in calls: + index = _index(call) if isinstance(call, dict) else None + if index is None or index in episode.poisoned: + continue + name = _function_name(call) + if agents_first and index in created: + # The stream continues this call here; the agent would start it over. + if name and not tool_allowed(name, self._allowlist): + self._refuse(index, name) + episode.poisoned.add(index) + episode.held.pop(index) + continue + if index in episode.held: + if name and not tool_allowed(name, self._allowlist): + self._refuse(index, name) + continue + if name: + episode.held[index] = name + elif _creates(call): + if not tool_allowed(name, self._allowlist): + self._refuse(index, name) + continue + episode.held[index] = name + created.add(index) + else: + continue # the agent ignores it + kept.append(call) + return [call for call in kept if _index(call) not in episode.poisoned] + + def _filter_first_choice(self, choice: dict) -> bool: + episode = self._episode + changed = False + delta = choice.get("delta") if isinstance(choice.get("delta"), dict) else None + finish = choice.get("finish_reason") + finish = finish if isinstance(finish, str) and finish else None # Goose reads "" as none + ends = episode.upstream and finish is not None + calls = delta.get("tool_calls") if delta is not None else None + if isinstance(calls, list) and calls: + if episode.upstream: + kept = self._later_chunk_calls(calls) + if kept and not episode.agent and finish not in (None, *_ENDS_AT_ONCE): + choice["finish_reason"] = "tool_calls" + changed = True + else: + self._episode = episode = _Episode(upstream=True) + kept = self._first_chunk_calls(calls) + ends = finish in _ENDS_AT_ONCE + episode.agent = episode.agent or bool(kept) + if len(kept) != len(calls): + changed = True + if kept: + delta["tool_calls"] = kept + else: + delta.pop("tool_calls") + notice = False + legacy = delta.get("function_call") if delta is not None else None + if isinstance(legacy, dict): + if self._legacy_allowed is None: + self._legacy_allowed = tool_allowed(legacy.get("name"), self._allowlist) + if not self._legacy_allowed: + self.withheld.append(str(legacy.get("name"))) + elif legacy.get("name") and not tool_allowed(legacy.get("name"), self._allowlist): + self._legacy_allowed = False + self.withheld.append(str(legacy.get("name"))) + if not self._legacy_allowed: + delta.pop("function_call") + changed = True + if finish == "function_call" and self._legacy_allowed is False: + choice["finish_reason"] = "stop" + notice = True + if episode.withheld and not episode.agent and not episode.noticed: + # The agent reads this chunk as plain content: tell the participant now, since + # the episode may end without a finish (a usage chunk, [DONE] or the end). + episode.noticed = notice = True + if ends: + if not episode.agent and episode.withheld and finish == "tool_calls": + choice["finish_reason"] = "stop" + changed = True + self._episode = _Episode() + if notice: + if delta is None: + delta = choice["delta"] = {} + content = delta.get("content") + if isinstance(content, list): + content.append({"type": "text", "text": ("\n\n" if content else "") + WITHHELD_NOTICE}) + elif isinstance(content, str) and content: + delta["content"] = content + "\n\n" + WITHHELD_NOTICE + elif not content: + delta["content"] = WITHHELD_NOTICE + changed = True + return changed + + def _drop_calls(self, choice: Any) -> bool: + """Goose reads only the first choice: calls in any other are removed.""" + delta = choice.get("delta") if isinstance(choice, dict) else None + if not isinstance(delta, dict) or not {"tool_calls", "function_call"} & delta.keys(): + return False + calls = delta.pop("tool_calls", None) + legacy = delta.pop("function_call", None) + names = [_function_name(call) for call in calls] if isinstance(calls, list) else [] + names.append(legacy.get("name") if isinstance(legacy, dict) else None) + self.withheld.extend( + name for name in names if isinstance(name, str) and name and not tool_allowed(name, self._allowlist) + ) + if choice.get("finish_reason") in ("tool_calls", "function_call"): + choice["finish_reason"] = "stop" + return True diff --git a/src/agents/tools.py b/src/agents/tools.py new file mode 100644 index 0000000..fde2cbb --- /dev/null +++ b/src/agents/tools.py @@ -0,0 +1,417 @@ +"""Catalogue of tool names an agent profile can select. + +This backs ``GET /api/agent/available-tools``, which the admin UI uses to +populate the tool picker when creating a profile, and it is the fallback +allowlist the proxy filters against for tasks whose profile predates per-profile +tool selection. + +Two runtimes contribute names: + +* ``code4me2-agent`` — the built-in ReAct loop, whose tools are implemented in + ``code4me2_agent/file_tools.py`` and ``command_tools.py``. This list is + authoritative, because we own that code. +* ``goose`` — Block's Goose advertises its own extension tools over ACP. The + names below are the ones observed in practice; the set genuinely varies with + the installed Goose build and its enabled extensions, so it is a *best-known* + catalogue rather than a contract. Anything Goose sends that isn't listed + shows up in the ``tools_stripped`` count on the model_call event, which is how + you discover a name that needs adding. + +Codex contributes nothing here: it manages its own tool schemas over the +Responses API, and the proxy passes them through rather than filtering them +(see ``agents.normalize.normalize_responses_api_body``). + +The module also owns the vocabulary and validation of the built-in runtime's +command and harness profile fields (decision D-01: ``commands_denylist``, +``command_timeout_seconds``, ``harness_options``), shared by the profile API, +the profile↔release contract, study freezing and the managed run policy. +""" + +from __future__ import annotations + +import re +from typing import Any, Mapping, Optional, Sequence + +# Built-in code4me2-agent tools. Each is enforced inside the runtime itself +# (paths confined to the workspace, commands checked against the profile's +# blocked commands). +CODE4ME2_AGENT_TOOLS: frozenset[str] = frozenset( + { + # Reading and discovery + "read_file", + "list_files", + "glob_files", + "grep_files", + "search_files", # legacy literal search; kept for frozen profiles + # Editing + "create_file", + "write_file", + "replace_text", + "edit_file", + "apply_patch", # multi-file patch, applied atomically + "delete_file", + "move_file", + # Interaction: ends the turn with a structured question to the user. + "ask_user", + # Execution and planning + "run_command", + "update_plan", + # Explicit opt-in for any tool exposed by client-supplied ACP MCP servers. + # Concrete names remain filtered and approval-gated by the managed runtime. + "mcp__*", + } +) + +# Goose tool names observed over ACP. Grouped by the extension that provides +# them, since enabling/disabling a Goose extension moves a whole group. +GOOSE_TOOLS: frozenset[str] = frozenset( + { + # developer extension + "shell", + "read", + "write", + "tree", + "edit", + "analyze", + # Goose 1.51's developer extension also offers this (observed in the + # study gateway shape, 2026-09-27). + "read_image", + # Builds that expose the developer extension as a namespaced MCP + # server name its tools like this. + "developer__shell", + "developer__text_editor", + # extension manager + "extensionmanager__list_resources", + "extensionmanager__manage_extensions", + "extensionmanager__read_resource", + "extensionmanager__search_available_extensions", + # apps extension + "apps__create_app", + "apps__delete_app", + "apps__iterate_app", + "apps__list_apps", + # misc built-ins + "delegate", + "load", + "load_skill", + "todo__todo_write", + } +) + +# Union of everything a profile may legitimately name. +KNOWN_AGENT_TOOLS: frozenset[str] = CODE4ME2_AGENT_TOOLS | GOOSE_TOOLS + +# Per-runtime view, for a UI that wants to show only the relevant tools once a +# framework_version has been picked. +TOOLS_BY_FRAMEWORK: dict[str, frozenset[str]] = { + "code4me2-agent": CODE4ME2_AGENT_TOOLS, + "goose": GOOSE_TOOLS, + # Codex tools are self-managed and passed through untouched. + "codex": frozenset(), +} + + +def tools_for_framework(framework_version: str | None) -> frozenset[str]: + """Tools selectable for a runtime; the full catalogue when unrecognised.""" + if not framework_version: + return KNOWN_AGENT_TOOLS + return TOOLS_BY_FRAMEWORK.get(framework_version.strip().lower(), KNOWN_AGENT_TOOLS) + + +# ── Built-in runtime command and harness settings (decision D-01) ──────────── +# +# Optional ``code4me2-agent`` profile fields; a BYOA (goose/codex) release +# refuses all three. ``None`` always means "not set": no command is blocked (a +# user's config row may still block some), and the runtime's default command +# timeout and harness defaults apply. Every set value is frozen with the +# profile into study snapshots and travels in ``GET /api/acp/agent-config`` and +# the managed run policy. +# +# ``commands_denylist`` replaced ``commands_allowlist``: the agent may run any +# program except the listed ones (the participant still approves each command +# under per-step approval). Study snapshots frozen before the change can still +# carry ``commands_allowlist``; it only reaches runtime releases 0.0.6 and +# earlier, which predate the denylist (see ``backend/routers/acp``). + +#: Profile fields, in the canonical order they join digests and snapshots. +HARNESS_PROFILE_FIELDS: tuple[str, ...] = ( + "commands_denylist", + "command_timeout_seconds", + "harness_options", +) + +#: A bare command name, as a denylist entry or a verify command's program: no +#: directory separator, whitespace or shell metacharacter (listing ``gradlew`` +#: also covers ``./gradlew``). Matched with ``fullmatch``, so a trailing newline +#: never slips through. +COMMAND_NAME_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+-]{0,63}") +COMMANDS_DENYLIST_MAX_ENTRIES = 64 +COMMANDS_ALLOWLIST_MAX_ENTRIES = 64 +COMMAND_TIMEOUT_SECONDS_MIN = 1 +COMMAND_TIMEOUT_SECONDS_MAX = 600 +VERIFY_COMMAND_MAX_ARGS = 32 +VERIFY_COMMAND_MAX_ARG_LENGTH = 512 + +#: ``harness_options.prompt_profile`` values (``auto`` derives the family from +#: the model name at runtime). +HARNESS_PROMPT_PROFILES: tuple[str, ...] = ( + "auto", + "default", + "openai", + "anthropic", + "gemini", +) +#: Boolean ``harness_options`` switches; the runtime default of each is true. +HARNESS_BOOLEAN_OPTIONS: tuple[str, ...] = ( + "self_review", + "verify_on_stop", + "context_summarization", + "parallel_tools", + "project_instructions", + "read_before_edit", + "syntax_check", + "loop_guard", + "instruction_reminders", + "test_output_summary", + "approval_revise", +) +#: Every key ``harness_options`` may carry; any other key is refused. +HARNESS_OPTION_KEYS: tuple[str, ...] = HARNESS_BOOLEAN_OPTIONS + ( + "verify_command", + "prompt_profile", +) + +_BARE_COMMAND_RULE = ( + "a bare command name (letters, digits, '.', '_', '+' or '-', at most 64 " + "characters, starting with a letter or digit; no path, spaces or shell " + "metacharacters)" +) + + +def _shown(value: Any) -> str: + """A short ``repr`` of a rejected value for an error message.""" + text = repr(value) + return text if len(text) <= 40 else text[:37] + "..." + + +def is_bare_command_name(value: Any) -> bool: + """Whether ``value`` is a bare command name (a program, not a path).""" + return isinstance(value, str) and COMMAND_NAME_PATTERN.fullmatch(value) is not None + + +# Blocked-command matching. Mirrors ``blocked_program`` in +# ``code4me2_agent/command_tools.py``, which enforces it in the runtime; keep +# the two in step. + +#: Launcher suffixes ignored when names are compared (``git.exe`` is ``git``). +_LAUNCHER_SUFFIXES = (".exe", ".cmd", ".bat", ".com") +#: Shells, launchers and package managers: every word of their arguments is +#: checked too, so ``bash -c "git push"``, ``env git push`` or ``npm x git`` cannot +#: start a blocked ``git``. A package manager's subcommands and aliases that run +#: a program are many (``uv run``, ``npm exec``/``x``/``explore``, ``mise x``), so +#: all its arguments count: a package named like a blocked program is refused too. +_COMMAND_RUNNERS = frozenset( + { + "bash", "sh", "zsh", "dash", "ksh", "fish", "csh", "tcsh", "busybox", + "cmd", "powershell", "pwsh", "wsl", + "env", "xargs", "sudo", "doas", "nohup", "nice", "ionice", "setsid", + "stdbuf", "timeout", "gtimeout", "time", "watch", "command", "exec", + "script", "arch", "caffeinate", "chroot", "su", "nix-shell", "npx", "uvx", "bunx", + "uv", "poetry", "pipenv", "pdm", "hatch", "rye", "pixi", "conda", "mamba", + "micromamba", "pipx", "npm", "pnpm", "yarn", "bun", "bundle", "corepack", + "mise", "asdf", "direnv", "pyenv", "rbenv", "nodenv", "nvm", "volta", + } +) +#: Quotes only group a word (``g''it`` is ``git``); the rest separate words, +#: including ``@`` before a version (``npx npm@10``, ``corepack pnpm@9``). +_QUOTES = re.compile(r"[\'\"]") +_WORD_SEPARATORS = re.compile(r"[\s;&|()<>`{}$=,@]+") + + +def command_key(name: str) -> str: + """The comparable program name: last path component, lower-cased, no ``.exe``. + + Trailing dots and spaces are dropped, as Windows drops them (``git.exe.`` + starts ``git.exe``). + """ + base = re.split(r"[\\/]", name.strip().rstrip("\\/"))[-1].lower().rstrip(" .") + for suffix in _LAUNCHER_SUFFIXES: + if base.endswith(suffix) and len(base) > len(suffix): + return base[: -len(suffix)].rstrip(" .") + return base + + +def blocked_command_in(argv: Sequence[str], denylist: Sequence[str]) -> Optional[str]: + """The blocked program ``argv`` would start, else ``None``. + + ``argv[0]`` is compared by name (case-insensitive, any directory and + launcher suffix ignored). When it is a shell, a launcher or a package + manager, every plainly written word of its arguments is compared as well + (``npm@10`` counts as ``npm``). Programs started by scripts, interpreters + or build tools, and names disguised with escapes, are not visible here. + """ + blocked = {command_key(name) for name in denylist} + blocked.discard("") + if not argv or not blocked: + return None + program = command_key(argv[0]) + if program in blocked: + return program + if program not in _COMMAND_RUNNERS: + return None + for argument in argv[1:]: + for word in _WORD_SEPARATORS.split(_QUOTES.sub("", argument)): + if word and command_key(word) in blocked: + return command_key(word) + return None + + +def validate_commands_denylist(value: Any) -> list[str]: + """Return a valid ``commands_denylist`` as a new list, else ``ValueError``. + + At most 64 bare command names, unique ignoring case; the order is kept (it + is the order the runtime and the editor show). An empty list blocks nothing. + """ + if not isinstance(value, list): + raise ValueError("commands_denylist must be a list of command names") + if len(value) > COMMANDS_DENYLIST_MAX_ENTRIES: + raise ValueError( + f"commands_denylist may list at most {COMMANDS_DENYLIST_MAX_ENTRIES} commands" + ) + seen: set[str] = set() + for index, item in enumerate(value): + if not is_bare_command_name(item): + raise ValueError( + f"commands_denylist[{index}] ({_shown(item)}) must be {_BARE_COMMAND_RULE}" + ) + if item.lower() in seen: + raise ValueError(f"commands_denylist lists {item!r} more than once") + seen.add(item.lower()) + return list(value) + + +def validate_commands_allowlist(value: Any) -> list[str]: + """Return a valid legacy ``commands_allowlist`` as a new list, else ``ValueError``. + + Only study snapshots frozen before ``commands_denylist`` carry one; it is + checked so a malformed frozen value still fails closed. At most 64 unique + bare command names; an empty list is valid (explicitly no commands). + """ + if not isinstance(value, list): + raise ValueError("commands_allowlist must be a list of command names") + if len(value) > COMMANDS_ALLOWLIST_MAX_ENTRIES: + raise ValueError( + f"commands_allowlist may list at most {COMMANDS_ALLOWLIST_MAX_ENTRIES} commands" + ) + seen: set[str] = set() + for index, item in enumerate(value): + if not is_bare_command_name(item): + raise ValueError( + f"commands_allowlist[{index}] ({_shown(item)}) must be {_BARE_COMMAND_RULE}" + ) + if item in seen: + raise ValueError(f"commands_allowlist lists {item!r} more than once") + seen.add(item) + return list(value) + + +def validate_command_timeout_seconds(value: Any) -> int: + """Return a valid ``command_timeout_seconds`` (an int in 1..600), else ``ValueError``.""" + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not COMMAND_TIMEOUT_SECONDS_MIN <= value <= COMMAND_TIMEOUT_SECONDS_MAX + ): + raise ValueError( + "command_timeout_seconds must be a whole number of seconds from " + f"{COMMAND_TIMEOUT_SECONDS_MIN} to {COMMAND_TIMEOUT_SECONDS_MAX}" + ) + return value + + +def validate_verify_command(value: Any) -> list[str]: + """Return a valid ``harness_options.verify_command`` argv, else ``ValueError``.""" + if not isinstance(value, list) or not 1 <= len(value) <= VERIFY_COMMAND_MAX_ARGS: + raise ValueError( + "harness_options.verify_command must be null or a list of 1 to " + f"{VERIFY_COMMAND_MAX_ARGS} arguments" + ) + for index, argument in enumerate(value): + if ( + not isinstance(argument, str) + or not argument + or len(argument) > VERIFY_COMMAND_MAX_ARG_LENGTH + or "\x00" in argument + ): + raise ValueError( + f"harness_options.verify_command[{index}] must be a non-empty string " + f"of at most {VERIFY_COMMAND_MAX_ARG_LENGTH} characters" + ) + if not is_bare_command_name(value[0]): + raise ValueError( + f"harness_options.verify_command[0] ({_shown(value[0])}) must be " + f"{_BARE_COMMAND_RULE}" + ) + return list(value) + + +def validate_harness_options( + value: Any, + *, + commands_denylist: Optional[Sequence[str]] = None, +) -> dict[str, Any]: + """Return a valid ``harness_options`` object as a new dict, else ``ValueError``. + + Only the keys in :data:`HARNESS_OPTION_KEYS` are accepted: booleans for the + switches, one of :data:`HARNESS_PROMPT_PROFILES` for ``prompt_profile`` and + ``null`` or an argv for ``verify_command``. Given the profile's + ``commands_denylist`` (the profile contract), the verify command must not + start a blocked program. A run policy snapshot is checked without it: a + user's config row may block more, and the runtime then skips the command. + """ + if not isinstance(value, Mapping): + raise ValueError("harness_options must be an object") + unknown = sorted(str(key) for key in value if key not in HARNESS_OPTION_KEYS) + if unknown: + raise ValueError( + "harness_options has unknown keys: " + + ", ".join(unknown) + + "; allowed keys are " + + ", ".join(HARNESS_OPTION_KEYS) + ) + for key in HARNESS_BOOLEAN_OPTIONS: + if key in value and not isinstance(value[key], bool): + raise ValueError(f"harness_options.{key} must be true or false") + if "prompt_profile" in value and ( + not isinstance(value["prompt_profile"], str) + or value["prompt_profile"] not in HARNESS_PROMPT_PROFILES + ): + raise ValueError( + "harness_options.prompt_profile must be one of " + + ", ".join(HARNESS_PROMPT_PROFILES) + ) + verify_command = value.get("verify_command") + if verify_command is not None: + validate_verify_command(verify_command) + blocked = blocked_command_in(verify_command, commands_denylist or ()) + if blocked is not None: + raise ValueError( + f"harness_options.verify_command runs {blocked!r}, which is in the " + "profile's commands_denylist" + ) + return dict(value) + + +def set_harness_profile_fields(profile: Any) -> dict[str, Any]: + """The D-01 fields ``profile`` sets, in :data:`HARNESS_PROFILE_FIELDS` order. + + Unset (``None``) fields are omitted, so a digest or snapshot built from a + profile without them keeps exactly the shape it had before they existed. + Works for ORM rows, frozen configs and stand-ins alike. + """ + values: dict[str, Any] = {} + for field in HARNESS_PROFILE_FIELDS: + value = getattr(profile, field, None) + if value is not None: + values[field] = value + return values diff --git a/src/backend/Responses.py b/src/backend/Responses.py index a7d024e..1f3ba82 100644 --- a/src/backend/Responses.py +++ b/src/backend/Responses.py @@ -1,9 +1,10 @@ from abc import ABC +from typing import Any, Optional from uuid import UUID from fastapi.encoders import jsonable_encoder from fastapi.responses import HTMLResponse, JSONResponse -from pydantic import BaseModel, Field, PrivateAttr +from pydantic import BaseModel, Field, PrivateAttr, model_serializer from response_models import ( ResponseCompletionResponseData, @@ -66,10 +67,6 @@ class UserAlreadyExistsWithThisEmail(ErrorResponse): message: str = Field(default="User already exists with this email!") -class InvalidOrExpiredJWTToken(ErrorResponse): - message: str = Field(default="Invalid or expired token!") - - class CreateUserError(ErrorResponse): message: str = Field(default="Server failed to create a new user!") @@ -86,10 +83,6 @@ class AuthenticateUserNormalPostResponse(AuthenticateUserPostResponse): ) -class AuthenticateUserOAuthPostResponse(AuthenticateUserPostResponse): - message: str = Field(default="User authenticated successfully via OAuth.") - - class InvalidEmailOrPassword(ErrorResponse): message: str = Field(default="Invalid email or password!") @@ -111,6 +104,12 @@ class DeleteUserError(ErrorResponse): message: str = Field(default="Server failed to delete the user!") +class DeleteUserBlockedError(ErrorResponse): + message: str = Field( + default="This account owns research studies or agent profiles, so it cannot delete itself." + ) + + class InvalidOrExpiredSessionToken(ErrorResponse): message: str = Field( default="Session not found! You are not authenticated or your session has expired. " @@ -236,6 +235,145 @@ class ProjectNotFoundError(ErrorResponse): ) +# /api/acp — grant handoff for a locally launched agent process +class PrepareAcpGrantPostResponse(BaseResponse): + message: str = Field(default="ACP agent session prepared successfully.") + grant: str = Field(..., description="One-time ACP launch grant.") + workspace: str = Field(..., description="Authorized canonical workspace.") + expires_in_seconds: int = Field(default=300, description="Grant lifetime.") + launch_id: Optional[str] = Field(default=None, description="Managed launch identity.") + path_format: Optional[str] = Field(default=None, description="Client path syntax.") + managed_protocol_version: Optional[str] = Field( + default=None, description="Negotiated managed-agent protocol." + ) + + +class ExchangeAcpGrantPostResponse(BaseResponse): + message: str = Field(default="ACP grant exchanged successfully.") + acp_token: str = Field(..., description="ACP process authorization token.") + project_id: str = Field(..., description="Authorized project id.") + workspace: str = Field(..., description="Authorized canonical workspace.") + expires_in_seconds: int = Field(default=3600, description="ACP session lifetime.") + + +class ValidateAcpSessionPostResponse(BaseResponse): + message: str = Field(default="ACP authorization validated successfully.") + project_id: str = Field(..., description="Authorized project id.") + workspace: str = Field(..., description="Authorized canonical workspace.") + expires_in_seconds: int = Field( + default=3600, description="Renewed ACP session lifetime." + ) + + +class AcpAgentConfigGetResponse(BaseResponse): + """Runtime configuration handed to a locally launched agent process. + + This is how the assigned A/B profile reaches the built-in runtime: rather + than the agent reading a local config file, the backend tells it which + model, provider endpoint, tools and step budget to use — so a study + assignment actually takes effect. + + ``api_key_ref`` is the *name* of an environment variable, never a key + value: the agent runs on the developer's machine and resolves the + credential from its own environment, so no secret transits this endpoint. + """ + + message: str = Field(default="ACP agent configuration retrieved successfully.") + agent_profile: Optional[str] = Field( + default=None, description="Assigned agent profile name." + ) + framework_version: Optional[str] = Field( + default=None, description="Agent runtime this profile targets." + ) + model: str = Field(description="Model the agent should use for completions.") + base_url: Optional[str] = Field( + default=None, + description="OpenAI-compatible provider base URL. Null = agent default.", + ) + api_key_ref: Optional[str] = Field( + default=None, + description="Name of the environment variable holding the provider API key.", + ) + commands_allowlist: list[str] = Field( + default_factory=list, + description="Legacy: the only commands runtime releases 0.0.6 and earlier " + "may execute. Newer runtimes ignore it and read commands_denylist.", + ) + commands_denylist: list[str] = Field( + default_factory=list, + description="Programs the agent may not run; every other installed " + "program may run. Empty = nothing blocked.", + ) + tools: list[str] = Field( + default_factory=list, description="Tool names the agent may call." + ) + max_iterations: int = Field( + default=8, description="Maximum tool-calling iterations.", ge=1 + ) + max_context_tokens: Optional[int] = Field( + default=None, description="Per-turn context window; null = model maximum." + ) + approval_policy: Optional[str] = Field( + default=None, description="How tool calls should be confirmed." + ) + temperature: Optional[float] = Field( + default=None, description="Sampling temperature; null = provider default." + ) + system_prompt: Optional[str] = Field( + default=None, + description="Researcher-authored system prompt frozen with the assigned " + "profile; null = none (the runtime keeps its built-in prompt).", + ) + command_timeout_seconds: Optional[int] = Field( + default=None, + description="Default per-command timeout in seconds frozen with the " + "assigned profile. Present only when the profile sets it; absent = the " + "runtime default.", + ) + harness_options: Optional[dict[str, Any]] = Field( + default=None, + description="Harness behaviour switches frozen with the assigned profile. " + "Present only when the profile sets them; absent = runtime defaults.", + ) + store_agent_content: bool = Field( + default=True, + description="Whether the server will persist content for this user. " + "Advisory only — the server enforces this regardless of what the agent " + "sends, so the agent may use it to avoid transmitting content " + "needlessly but cannot use it to enable capture.", + ) + transport: Optional[str] = Field( + default=None, description="Managed inference transport selected by the server." + ) + managed_protocol_version: Optional[str] = Field( + default=None, description="Negotiated managed-agent protocol." + ) + + @model_serializer(mode="wrap") + def _omit_unset_harness_settings(self, handler): + # A config whose profile sets neither harness field serializes exactly + # as it did before the fields existed (decision D-01). + data = handler(self) + if isinstance(data, dict): + for key in _OPTIONAL_HARNESS_CONFIG_KEYS: + if data.get(key) is None: + data.pop(key, None) + return data + + +#: ``AcpAgentConfigGetResponse`` keys omitted (not null) when unset. +_OPTIONAL_HARNESS_CONFIG_KEYS = ("command_timeout_seconds", "harness_options") + + +class PersistentAuthTokenPostResponse(BaseResponse): + message: str = Field(default="Persistent auth token created successfully.") + auth_token: str = Field(..., description="Auth token without Redis expiration.") + + +class AcpAuthorizationError(ErrorResponse): + message: str = Field(default="ACP authorization is invalid or expired.") + + # /api/session/deactivate class DeactivateSessionPostResponse(BaseResponse): message: str = Field(default="Session deactivated successfully.") @@ -275,6 +413,12 @@ class TooManyRequests(ErrorResponse): message: str = Field(default="Too many requests. Please try again later.") +class ClassicModelsDisabled(ErrorResponse): + message: str = Field( + default="Classic completion and chat models are disabled on this server." + ) + + # /api/chat/delete class DeleteChatError(ErrorResponse): """Error response for chat deletion failure.""" diff --git a/src/backend/acp_authorization.py b/src/backend/acp_authorization.py new file mode 100644 index 0000000..07d195d --- /dev/null +++ b/src/backend/acp_authorization.py @@ -0,0 +1,402 @@ +"""Authorization for locally-launched ACP agent processes. + +The built-in ``code4me2-agent`` runtime runs as a separate OS process on the +developer's machine, started by the IDE plugin. It therefore has no session +cookie and cannot be authenticated the way the plugin is. This module implements +the handoff that gives it a credential of its own: + +1. **grant** — the authenticated plugin asks for a one-time launch grant, bound + to a specific user, session, project and workspace directory. The plugin + writes it where the agent process will find it. +2. **exchange** — the agent process trades the grant (single-use) for a longer + lived ``acp_session`` token. Consuming the grant means a leaked handoff file + can't be replayed after the agent has started. +3. **validate-or-refresh** — every subsequent agent request revalidates and + slides the session's expiry, so an idle agent expires but an active one + doesn't. + +Every scope is *derived*, never accepted from the caller. An agent request can +only ever act as the user, project and workspace that were bound into the grant +at step 1, and each revalidation re-checks that the parent plugin login and +session are still alive — so logging out in the IDE immediately invalidates the +agent process too, rather than leaving an orphaned credential valid for an hour. +""" + +from __future__ import annotations + +import secrets +from dataclasses import dataclass +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Any, Callable, Optional, Protocol, Union + + +class TokenStore(Protocol): + """The subset of ``RedisManager`` this service needs. + + Declared as a Protocol so the authorization logic is testable against an + in-memory dict without a Redis instance. + """ + + def get( + self, type: str, token: str, reset_exp: bool = False + ) -> Optional[dict]: ... + + def set( + self, + type: str, + token: str, + info: dict, + force_reset_exp: bool = False, + ) -> None: ... + + def consume(self, type: str, token: str) -> Optional[dict]: ... + + def discard(self, type: str, token: str) -> None: ... + + +class AcpAuthorizationDenied(Exception): + """Raised for every authorization failure in this module. + + Deliberately carries a coarse message: the caller maps it to a flat 401 so + the response doesn't reveal *which* check failed. + """ + + +@dataclass(frozen=True) +class PreparedAcpGrant: + grant: str + workspace: str + launch_id: Optional[str] = None + path_format: Optional[str] = None + managed_protocol_version: Optional[str] = None + expires_in_seconds: int = 300 + + +@dataclass(frozen=True) +class AcpSessionAuthorization: + acp_token: str + user_id: str + project_id: str + workspace: str + expires_in_seconds: int = 3600 + + +@dataclass(frozen=True) +class AcpServerAuthorization: + """A session scope plus the resolved project record. + + Needed by endpoints that act on project data (e.g. bridging a model turn + through the backend's own model registry), where the project's multi-file + context has to be loaded. + """ + + acp_token: str + user_id: str + session_id: str + project_id: str + project_info: dict[str, Any] + workspace: str + expires_in_seconds: int = 3600 + + +def authorize_acp_bearer( + token_store: TokenStore, + authorization: str, + *, + server_scope: bool = False, +) -> Optional[Union[AcpSessionAuthorization, AcpServerAuthorization]]: + """Resolve an ``Authorization: Bearer `` header to a scope. + + Returns None on any failure so route handlers can respond with a single + uniform 401 rather than branching on the reason. + """ + prefix = "Bearer " + if not authorization.startswith(prefix) or not authorization[len(prefix) :]: + return None + service = AcpAuthorizationService(token_store) + try: + if server_scope: + return service.resolve_server_scope(authorization[len(prefix) :]) + return service.validate_or_refresh(authorization[len(prefix) :]) + except AcpAuthorizationDenied: + return None + + +class AcpAuthorizationService: + def __init__( + self, + token_store: TokenStore, + *, + token_factory: Optional[Callable[[], str]] = None, + ) -> None: + self._tokens = token_store + self._token_factory = token_factory or (lambda: secrets.token_urlsafe(32)) + + def prepare_grant( + self, + *, + auth_token: str, + project_id: str, + workspace: str, + launch_id: Optional[str] = None, + path_format: Optional[str] = None, + managed_protocol_version: Optional[str] = None, + ) -> PreparedAcpGrant: + """Mint a one-time launch grant for an authenticated plugin. + + Any grant previously issued for this plugin session is discarded first, + so re-preparing (which the plugin does on every startup and login) + leaves exactly one usable grant rather than accumulating valid tokens. + """ + managed_values = (launch_id, path_format, managed_protocol_version) + if any(value is not None for value in managed_values) and not all( + value is not None for value in managed_values + ): + raise AcpAuthorizationDenied( + "Managed launch identity, path format, and protocol are required together." + ) + if managed_protocol_version is not None and managed_protocol_version != "1": + raise AcpAuthorizationDenied("Unsupported managed protocol version.") + canonical_workspace = ( + self._validate_client_workspace(workspace, path_format) + if path_format is not None + else self._canonical_workspace(workspace) + ) + state = self._active_plugin_scope( + auth_token=auth_token, + project_id=project_id, + workspace=canonical_workspace, + already_canonical=True, + ) + if launch_id is not None: + state["launch_id"] = launch_id + if path_format is not None: + state["path_format"] = path_format + if managed_protocol_version is not None: + state["managed_protocol_version"] = managed_protocol_version + pending_key = self._pending_grant_key(state, launch_id) + pending = self._tokens.get("acp_pending_grant", pending_key) + if pending is not None and pending.get("grant"): + pending_grant = pending["grant"] + pending_state = self._tokens.get("acp_grant", pending_grant) + if launch_id is not None and pending_state == { + **state, + "pending_grant_key": pending_key, + }: + # A bridge request can be retried after its HTTP response is + # lost. Return the still-live grant for this exact launch so + # two concurrent responses cannot invalidate each other. + return PreparedAcpGrant( + grant=pending_grant, + workspace=state["workspace"], + launch_id=launch_id, + path_format=path_format, + managed_protocol_version=managed_protocol_version, + ) + self._tokens.discard("acp_grant", pending_grant) + grant = self._token_factory() + state["pending_grant_key"] = pending_key + self._tokens.set("acp_grant", grant, state, force_reset_exp=True) + self._tokens.set( + "acp_pending_grant", + pending_key, + {"grant": grant}, + force_reset_exp=True, + ) + return PreparedAcpGrant( + grant=grant, + workspace=state["workspace"], + launch_id=launch_id, + path_format=path_format, + managed_protocol_version=managed_protocol_version, + ) + + def exchange_grant(self, grant: str) -> AcpSessionAuthorization: + """Trade a launch grant for an agent session token. + + The scope is re-checked *before* consuming, then the grant is consumed + atomically — so a grant whose parent session died in the meantime is + rejected, and a valid grant can only ever be redeemed once. + """ + state = self._tokens.get("acp_grant", grant) + if state is None: + raise AcpAuthorizationDenied( + "ACP grant is invalid or has already been used." + ) + self._require_active_scope(state) + consumed_state = self._tokens.consume("acp_grant", grant) + if consumed_state is None: + raise AcpAuthorizationDenied( + "ACP grant is invalid or has already been used." + ) + pending_key = consumed_state.get("pending_grant_key") + if pending_key: + self._tokens.discard("acp_pending_grant", pending_key) + acp_token = self._token_factory() + self._tokens.set("acp_session", acp_token, consumed_state, force_reset_exp=True) + return AcpSessionAuthorization( + acp_token=acp_token, + user_id=consumed_state["user_id"], + project_id=consumed_state["project_id"], + workspace=consumed_state["workspace"], + ) + + def validate_or_refresh(self, acp_token: str) -> AcpSessionAuthorization: + state = self._validated_session_state(acp_token) + return AcpSessionAuthorization( + acp_token=acp_token, + user_id=state["user_id"], + project_id=state["project_id"], + workspace=state["workspace"], + ) + + def resolve_server_scope(self, acp_token: str) -> AcpServerAuthorization: + state = self._validated_session_state(acp_token) + project_info = self._tokens.get("project_token", state["project_id"]) + if project_info is None: + raise AcpAuthorizationDenied("An active project association is required.") + return AcpServerAuthorization( + acp_token=acp_token, + user_id=state["user_id"], + session_id=state["parent_session_token"], + project_id=state["project_id"], + project_info=project_info, + workspace=state["workspace"], + ) + + def _validated_session_state(self, acp_token: str) -> dict[str, str]: + """Revalidate an agent session and slide its expiry. + + A session whose parent plugin login has gone away is *discarded* here, + not merely rejected — otherwise it would sit in Redis until its TTL, + still resolvable if the user logged back in. + """ + state = self._tokens.get("acp_session", acp_token) + if state is None: + raise AcpAuthorizationDenied("ACP session is invalid or expired.") + try: + self._require_active_scope(state) + except AcpAuthorizationDenied: + self._tokens.discard("acp_session", acp_token) + raise + # Touch the parent session so an actively-working agent keeps the user's + # IDE session alive, matching how plugin requests behave. + self._tokens.get("session_token", state["parent_session_token"], reset_exp=True) + self._tokens.set("acp_session", acp_token, state, force_reset_exp=True) + return state + + def _active_plugin_scope( + self, + *, + auth_token: str, + project_id: str, + workspace: str, + already_canonical: bool = False, + ) -> dict[str, str]: + auth_info = self._tokens.get("auth_token", auth_token) + if auth_info is None or not auth_info.get("user_id"): + raise AcpAuthorizationDenied("An authenticated plugin login is required.") + user_id = auth_info["user_id"] + user_info = self._tokens.get("user_token", user_id) + if user_info is None or not user_info.get("session_token"): + raise AcpAuthorizationDenied("An active plugin session is required.") + # Managed grants arrive with a client-canonicalized workspace (validated + # against the participant OS without touching the server filesystem). + # Re-resolving that value with the server OS would reject Windows drive + # paths on a Linux backend, so already-canonical values are preserved. + canonical = workspace if already_canonical else self._canonical_workspace(workspace) + return self._bound_scope( + auth_token=auth_token, + user_id=user_id, + session_token=user_info["session_token"], + project_id=project_id, + workspace=canonical, + ) + + def _require_active_scope(self, state: dict[str, str]) -> None: + """Re-check that everything the grant was bound to is still valid.""" + auth_info = self._tokens.get("auth_token", state["parent_auth_token"]) + if auth_info is None or auth_info.get("user_id") != state["user_id"]: + raise AcpAuthorizationDenied("The parent plugin login is no longer active.") + user_info = self._tokens.get("user_token", state["user_id"]) + if ( + user_info is None + or user_info.get("session_token") != state["parent_session_token"] + ): + raise AcpAuthorizationDenied( + "The parent plugin session is no longer active." + ) + self._bound_scope( + auth_token=state["parent_auth_token"], + user_id=state["user_id"], + session_token=state["parent_session_token"], + project_id=state["project_id"], + workspace=state["workspace"], + ) + + def _bound_scope( + self, + *, + auth_token: str, + user_id: str, + session_token: str, + project_id: str, + workspace: str, + ) -> dict[str, str]: + """Verify the session↔project association in both directions. + + Checking only one direction would let a stale association on either side + authorize an agent against a project the session no longer has open. + """ + session_info = self._tokens.get("session_token", session_token) + if session_info is None or project_id not in session_info.get( + "project_tokens", [] + ): + raise AcpAuthorizationDenied("An active project association is required.") + project_info = self._tokens.get("project_token", project_id) + if project_info is None or session_token not in project_info.get( + "session_tokens", [] + ): + raise AcpAuthorizationDenied("An active project association is required.") + return { + "user_id": user_id, + "parent_auth_token": auth_token, + "parent_session_token": session_token, + "project_id": project_id, + "workspace": workspace, + } + + @staticmethod + def _canonical_workspace(workspace: str) -> str: + """Resolve the workspace to a canonical absolute path. + + The agent's file tools confine themselves to this directory, so it must + be normalized before being bound into the grant — otherwise a relative + or ``..``-containing path could widen the agent's reach. + """ + path = Path(workspace).expanduser() + if not path.is_absolute(): + raise AcpAuthorizationDenied("A canonical absolute workspace is required.") + return path.resolve().as_posix() + + @staticmethod + def _validate_client_workspace(workspace: str, path_format: str) -> str: + """Validate an already-canonical client path without using the server OS.""" + if path_format not in {"windows", "posix"}: + raise AcpAuthorizationDenied("Unsupported client path format.") + path_cls = PureWindowsPath if path_format == "windows" else PurePosixPath + path = path_cls(workspace) + if not path.is_absolute() or ".." in path.parts: + raise AcpAuthorizationDenied("A canonical absolute workspace is required.") + return str(path) + + @staticmethod + def _pending_grant_key(state: dict[str, str], launch_id: Optional[str]) -> str: + # Legacy callers retain the old one-pending-grant-per-plugin-session + # behavior. Managed callers isolate retries and concurrent project launches. + if launch_id is None: + return state["parent_session_token"] + return ( + f"{state['parent_session_token']}:{state['project_id']}:" + f"{state['workspace']}:{launch_id}" + ) diff --git a/src/backend/classic_models_gate.py b/src/backend/classic_models_gate.py new file mode 100644 index 0000000..aad7a91 --- /dev/null +++ b/src/backend/classic_models_gate.py @@ -0,0 +1,49 @@ +""" +Switch for the classic model-backed endpoints. + +The classic plugin surfaces (inline completion and chat) run the models listed in +``model_name``; a local row is downloaded and loaded on the first request that uses +it. With ``CLASSIC_MODELS_ENABLED=false`` (the no-GPU dev stack) those requests are +refused here, before any route or model code runs, so no model is ever loaded. +Every other endpoint, including completion feedback and chat history, is unaffected. +""" + +from starlette.types import ASGIApp, Receive, Scope, Send +from starlette.websockets import WebSocketClose + +from backend.Responses import ClassicModelsDisabled, JsonResponseWithStatus + +# The only routes that run a classic model; the WebSocket ones feed the Celery +# ``llm`` queue. +CLASSIC_MODEL_PATHS = frozenset( + { + "/api/completion/request", + "/api/chat/request", + "/api/ws/completion", + "/api/ws/chat", + } +) + +# WebSocket close code 1013 ("Try Again Later"): the server is up but does not +# serve this endpoint. +WEBSOCKET_CLOSE_CODE = 1013 + + +class ClassicModelsGate: + """ASGI middleware refusing the classic model endpoints (HTTP 503, WebSocket close).""" + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] not in ("http", "websocket") or ( + scope["path"].rstrip("/") not in CLASSIC_MODEL_PATHS + ): + await self.app(scope, receive, send) + return + + if scope["type"] == "http": + response = JsonResponseWithStatus(status_code=503, content=ClassicModelsDisabled()) + await response(scope, receive, send) + else: + await WebSocketClose(code=WEBSOCKET_CLOSE_CODE)(scope, receive, send) diff --git a/src/backend/completion/.env b/src/backend/completion/.env index fb62dbc..2bb153a 100644 --- a/src/backend/completion/.env +++ b/src/backend/completion/.env @@ -28,6 +28,10 @@ DB_PORT=2345 DB_USER=postgres DB_PASSWORD=postgres DB_NAME=code4meV2 +DB_POOL_SIZE=10 +DB_MAX_OVERFLOW=20 +DB_POOL_TIMEOUT=30 +DB_POOL_RECYCLE=1800 # ----------------------- # pgAdmin Configuration @@ -83,4 +87,4 @@ THREAD_POOL_MAX_WORKERS=2 # ----------------------- # Hugging Face Configuration HF_TOKEN=your_huggingface_token_here -# ----------------------- \ No newline at end of file +# ----------------------- diff --git a/src/backend/completion/OpenAICompatibleModel.py b/src/backend/completion/OpenAICompatibleModel.py new file mode 100644 index 0000000..d6c3837 --- /dev/null +++ b/src/backend/completion/OpenAICompatibleModel.py @@ -0,0 +1,339 @@ +"""OpenAI-compatible provider-backed models for the classic endpoints. + +The classic chat (``POST /api/chat/request``) and completion +(``POST /api/completion/request``) endpoints normally run local HuggingFace +models. A ``model_name`` row can opt into an OpenAI-compatible provider +(OpenRouter, Ollama, OpenAI, Groq, vLLM) by carrying a ``model_parameters`` JSON +object with ``"provider": "openai_compatible"``; the row is then served over +HTTP instead of the local stack. + +Secrets follow the repository rule used by ``src/agents/provider.py``: the +configuration stores only the *name* of the environment variable that holds the +key (``api_key_ref``). The value is read from ``os.environ`` at request time and +is never stored, logged, or embedded in an error message. There is no fallback: +a missing key variable or an invalid configuration is a hard error. + +This module must stay free of torch/transformers imports so a torch-free image +can serve the classic endpoints. +""" + +from __future__ import annotations + +import os +import re +import time +from typing import Any, Dict, List, Literal, Optional + +import httpx +from langchain_core.messages import AIMessage, BaseMessage, SystemMessage +from pydantic import BaseModel, ConfigDict, Field, field_validator + +__all__ = [ + "OpenAICompatibleChatModel", + "OpenAICompatibleCompletionModel", + "OpenAICompatibleConfig", +] + +COMPLETION_SYSTEM_INSTRUCTION = ( + "You are a code completion engine. Continue the code at the cursor. " + "Return only the code that belongs at the cursor, with no explanations." +) + +_MAX_ERROR_EXCERPT_CHARS = 200 + +# Environment-variable names only; a value pasted here by mistake must be +# rejected without ever being echoed back. +_ENV_VAR_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +class OpenAICompatibleConfig(BaseModel): + """Per-row provider configuration parsed from ``model_name.model_parameters``. + + Unknown keys are rejected so a typo cannot silently change the wire shape. + ``base_url`` is required; ``api_key_ref`` is optional for providers that need + no authentication (e.g. a local Ollama). + """ + + model_config = ConfigDict(extra="forbid") + + provider: Literal["openai_compatible"] = "openai_compatible" + kind: Optional[Literal["chat", "completion"]] = None + base_url: str = Field(..., min_length=1) + api_key_ref: Optional[str] = None + provider_model: Optional[str] = None + endpoint: Literal["chat", "completions"] = "chat" + max_new_tokens: int = Field(default=256, ge=1) + temperature: float = Field(default=0.2, ge=0.0) + top_p: float = Field(default=0.95, gt=0.0, le=1.0) + timeout_seconds: float = Field(default=60.0, gt=0.0) + + @field_validator("base_url") + @classmethod + def _strip_base_url(cls, value: str) -> str: + stripped = value.strip() + if not stripped: + raise ValueError("base_url must not be empty") + return stripped + + def resolved_kind(self, model_name: str) -> Literal["chat", "completion"]: + """The row's kind, defaulting from the model name like the local loader.""" + if self.kind is not None: + return self.kind + return "chat" if "instruct" in model_name.lower() else "completion" + + def resolved_model(self, model_name: str) -> str: + """The provider-side model id; defaults to the row's ``model_name``.""" + return self.provider_model or model_name + + def resolve_api_key(self) -> Optional[str]: + """Read the provider key from the environment by variable *name*. + + Returns ``None`` when the row configures no ``api_key_ref``. A configured + but missing/empty variable is a hard error that names the variable; the + value is never stored or logged. + """ + env_name = (self.api_key_ref or "").strip() + if not env_name: + return None + if not _ENV_VAR_NAME.match(env_name): + # Never echo the value: an operator may have pasted the key itself. + raise RuntimeError( + "api_key_ref must be an environment variable name " + "(letters, digits and underscores)" + ) + key = os.getenv(env_name, "").strip() + if not key: + raise RuntimeError(f"provider API key environment variable ${env_name} is not set") + return key + + @property + def chat_completions_url(self) -> str: + return f"{self.base_url.rstrip('/')}/chat/completions" + + @property + def completions_url(self) -> str: + return f"{self.base_url.rstrip('/')}/completions" + + +def _post_json( + config: OpenAICompatibleConfig, url: str, payload: Dict[str, Any] +) -> Dict[str, Any]: + """POST a JSON payload and return the decoded JSON object. + + Raises ``RuntimeError`` for transport failures, non-2xx responses and + non-object JSON bodies. Messages never include the API key. + """ + headers = {"Content-Type": "application/json"} + api_key = config.resolve_api_key() + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + try: + response = httpx.post(url, json=payload, headers=headers, timeout=config.timeout_seconds) + except httpx.HTTPError as exc: + raise RuntimeError(f"provider request to {url} failed: {exc}") from exc + + if not 200 <= response.status_code < 300: + excerpt = " ".join(str(getattr(response, "text", "") or "").split()) + if api_key: + # Defensive: never surface the key even if an upstream echoes it. + excerpt = excerpt.replace(api_key, "[redacted]") + raise RuntimeError( + f"provider request to {url} failed with status {response.status_code}: " + f"{excerpt[:_MAX_ERROR_EXCERPT_CHARS]}" + ) + + try: + data = response.json() + except ValueError as exc: + raise RuntimeError(f"provider response from {url} was not valid JSON") from exc + if not isinstance(data, dict): + raise RuntimeError(f"provider response from {url} was not a JSON object") + return data + + +def _first_choice(data: Dict[str, Any], provider_model: str) -> Dict[str, Any]: + choices = data.get("choices") + if not isinstance(choices, list) or not choices: + raise RuntimeError(f"provider response for model {provider_model!r} contained no choices") + choice = choices[0] + if not isinstance(choice, dict): + raise RuntimeError( + f"provider response for model {provider_model!r} had a malformed choice" + ) + return choice + + +def _chat_message_content(choice: Dict[str, Any], provider_model: str) -> str: + """Extract ``choices[0].message.content``, failing clearly on malformed bodies.""" + message = choice.get("message") + if not isinstance(message, dict): + raise RuntimeError(f"provider response for model {provider_model!r} had no message object") + content = message.get("content") + return "" if content is None else str(content) + + +def _wire_role(message: BaseMessage) -> str: + """Map a LangChain message to an OpenAI chat role. + + ``SystemMessage``/``AIMessage``/``HumanMessage`` map to + ``system``/``assistant``/``user``; every other message type maps to ``user``. + """ + if isinstance(message, SystemMessage): + return "system" + if isinstance(message, AIMessage): + return "assistant" + return "user" + + +class OpenAICompatibleChatModel: + """Provider-backed chat model with the ``ChatCompletionModel.invoke`` contract. + + ``invoke(messages)`` returns ``completion``, ``generation_time``, ``role`` + and ``model_name`` exactly like the local model. + """ + + def __init__(self, model_name: str, config: OpenAICompatibleConfig) -> None: + self.model_name = model_name + self.config = config + self.provider_model = config.resolved_model(model_name) + + def invoke(self, messages: List[BaseMessage], **kwargs: Any) -> Dict[str, Any]: + t0 = time.perf_counter() + payload: Dict[str, Any] = { + "model": self.provider_model, + "messages": [ + {"role": _wire_role(message), "content": str(message.content)} + for message in messages + ], + "max_tokens": self.config.max_new_tokens, + "temperature": self.config.temperature, + "top_p": self.config.top_p, + } + data = _post_json(self.config, self.config.chat_completions_url, payload) + t1 = time.perf_counter() + + content = _chat_message_content( + _first_choice(data, self.provider_model), self.provider_model + ) + + return { + "completion": str(content).strip(), + "generation_time": int((t1 - t0) * 1000), + "role": "assistant", + "model_name": self.model_name, + } + + +class OpenAICompatibleCompletionModel: + """Provider-backed completion model with the local model's result shape. + + Prompt formatting mirrors ``TemplateCompletionModel._format_prompt_from_dict`` + (same multi-file/single-file template selection, ``file_separator`` semantics + and ``stop_tokens``) without importing torch/transformers or LangChain prompt + helpers. + """ + + def __init__( + self, + model_name: str, + prompt_templates: dict, + config: OpenAICompatibleConfig, + ) -> None: + self.model_name = model_name + self.prompt_templates = prompt_templates + self.config = config + self.provider_model = config.resolved_model(model_name) + self._validate_prompt_templates(prompt_templates) + + @staticmethod + def _validate_prompt_templates(prompt_templates: dict) -> None: + """Same shape checks as the local template model, so bad rows fail fast.""" + if not isinstance(prompt_templates, dict): + raise ValueError("prompt_templates must be a dictionary.") + if "fim_template" not in prompt_templates: + raise KeyError("prompt_templates must contain the 'fim_template' key.") + if "file_separator" not in prompt_templates: + raise KeyError("prompt_templates must contain the 'file_separator' key.") + fim_template = prompt_templates["fim_template"] + if not isinstance(fim_template, dict): + raise ValueError("'fim_template' must be a dictionary.") + if "single_file_template" not in fim_template: + raise KeyError("'fim_template' must contain the 'single_file_template' key.") + if "multi_file_template" not in fim_template: + raise KeyError("'fim_template' must contain the 'multi_file_template' key.") + + def _format_prompt_from_dict(self, prompt: dict) -> str: + """Build a formatted FIM prompt, mirroring the local template model.""" + if "multi_file_context" in prompt: + multi_file_context_prompt = "" + for file_name, file_code in prompt["multi_file_context"].items(): + multi_file_context_prompt += ( + self.prompt_templates["file_separator"].replace("{file_name}", file_name) + + file_code + + "\n" + ) + prompt = {**prompt, "multi_file_context": multi_file_context_prompt} + + template_key = ( + "multi_file_template" if "multi_file_context" in prompt else "single_file_template" + ) + return self.prompt_templates["fim_template"][template_key].format(**prompt) + + def invoke( + self, + prompt: dict, + max_new_tokens: Optional[int] = None, + stop_sequences: Optional[List[str]] = None, + **kwargs: Any, + ) -> dict: + """Generate a completion and return the local model's result shape. + + Returns ``completion``, ``generation_time``, ``confidence`` and + ``logprobs``. The provider path has no token logprobs, so ``confidence`` + is ``None`` (stored as NULL and left out of the calibration and model + analytics, which filter on ``confidence IS NOT NULL``) and ``logprobs`` + is empty. A fabricated ``0.0`` would skew every calibration chart. + """ + t0 = time.perf_counter() + formatted_prompt = self._format_prompt_from_dict(prompt) + stop = list(stop_sequences or []) + list(self.prompt_templates.get("stop_tokens") or []) + max_tokens = max_new_tokens if max_new_tokens is not None else self.config.max_new_tokens + + if self.config.endpoint == "completions": + payload: Dict[str, Any] = { + "model": self.provider_model, + "prompt": formatted_prompt, + "max_tokens": max_tokens, + "temperature": self.config.temperature, + "top_p": self.config.top_p, + } + if stop: + payload["stop"] = stop + data = _post_json(self.config, self.config.completions_url, payload) + completion = str(_first_choice(data, self.provider_model).get("text") or "") + else: + payload = { + "model": self.provider_model, + "messages": [ + {"role": "system", "content": COMPLETION_SYSTEM_INSTRUCTION}, + {"role": "user", "content": formatted_prompt}, + ], + "max_tokens": max_tokens, + "temperature": self.config.temperature, + "top_p": self.config.top_p, + } + if stop: + payload["stop"] = stop + data = _post_json(self.config, self.config.chat_completions_url, payload) + completion = _chat_message_content( + _first_choice(data, self.provider_model), self.provider_model + ) + + t1 = time.perf_counter() + return { + "completion": completion, + "generation_time": int((t1 - t0) * 1000), + "confidence": None, + "logprobs": [], + } diff --git a/src/backend/completion/__init__.py b/src/backend/completion/__init__.py index 333a2df..c36af02 100644 --- a/src/backend/completion/__init__.py +++ b/src/backend/completion/__init__.py @@ -1,14 +1,87 @@ -import logging -from typing import Optional, Union +from __future__ import annotations -import torch +import hashlib import json +import logging +from typing import TYPE_CHECKING, Optional, Union + +from pydantic import ValidationError + +from backend.completion.OpenAICompatibleModel import ( + OpenAICompatibleChatModel, + OpenAICompatibleCompletionModel, + OpenAICompatibleConfig, +) + +if TYPE_CHECKING: + from Code4meV2Config import Code4meV2Config + +try: + import torch +except ImportError: + torch = None + +if torch is not None: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + +# The completion model classes need the local torch/transformers stack. Import +# them at module level when it is present; otherwise expose placeholders so +# `backend.completion.` resolves and fails with a clear message only if +# something actually tries to use the local model stack in a torch-free image +# (e.g. the fast dev image, which always uses an external provider). +if torch is not None: + from backend.completion.ChatCompletionModel import ChatCompletionModel + from backend.completion.TemplateCompletionModel import TemplateCompletionModel +else: -torch.backends.cuda.matmul.allow_tf32 = True -torch.backends.cudnn.allow_tf32 = True -from backend.completion.ChatCompletionModel import ChatCompletionModel -from backend.completion.TemplateCompletionModel import TemplateCompletionModel -from Code4meV2Config import Code4meV2Config + class _TorchRequiredModel: + def __init__(self, *args, **kwargs): + raise RuntimeError( + "This image runs without the local Hugging Face/torch model stack; " + "use an external OpenAI-compatible provider instead." + ) + + ChatCompletionModel = _TorchRequiredModel # type: ignore[assignment,misc] + TemplateCompletionModel = _TorchRequiredModel # type: ignore[assignment,misc] + +# Accepted model types for `POST /api/chat/request`. Defined from the names above +# so it resolves in both torch-present and torch-free images. +CLASSIC_CHAT_MODEL_TYPES = (ChatCompletionModel, OpenAICompatibleChatModel) + + +def _provider_config_for( + model_parameters: str, model_name: str +) -> Optional[OpenAICompatibleConfig]: + """Return the provider config for a row that opts in, else ``None`` (local row). + + A row with ``"provider": "openai_compatible"`` is validated eagerly. Invalid + configuration raises a sanitized ``RuntimeError`` that names the model and the + offending fields but never echoes raw parameter values (which could contain a + mistakenly pasted secret). + """ + try: + params = json.loads(model_parameters) + except (TypeError, ValueError): + return None + if not isinstance(params, dict) or "provider" not in params: + return None + if params.get("provider") != "openai_compatible": + # A typo'd provider value must not silently fall back to a local model. + raise RuntimeError( + f"unsupported 'provider' value for model {model_name!r}; " + "only 'openai_compatible' is supported" + ) + try: + return OpenAICompatibleConfig(**params) + except ValidationError as exc: + details = "; ".join( + f"{'.'.join(str(part) for part in error['loc'])}: {error['msg']}" + for error in exc.errors() + ) + raise RuntimeError( + f"invalid openai_compatible configuration for model {model_name!r}: {details}" + ) from None class CompletionModels: @@ -41,6 +114,38 @@ def __init__(self, config: Code4meV2Config): self.__models = {} # Dictionary to store loaded models keyed by name/template. self._initialized = True + @staticmethod + def _model_cache_key( + model_name: str, prompt_templates: str, model_parameters: str + ) -> tuple[Optional[str], Optional[OpenAICompatibleConfig]]: + """Return ``(cache key, provider config)`` for a row. + + Provider rows use a key that includes kind, base_url and provider model so + distinct provider rows never collide. Returns ``(None, None)`` when the row + opts in but its configuration is invalid (the error is logged here). + """ + try: + provider_config = _provider_config_for(model_parameters, model_name) + except RuntimeError as exc: + logging.error(exc) + return None, None + if provider_config is not None: + fingerprint = hashlib.sha256( + provider_config.model_dump_json().encode("utf-8") + ).hexdigest()[:16] + key = ( + f"{model_name}:provider:{provider_config.resolved_kind(model_name)}:" + f"{provider_config.base_url}:{provider_config.resolved_model(model_name)}:" + f"{fingerprint}" + ) + else: + key = ( + f"{model_name}:instruct" + if "instruct" in model_name.lower() + else f"{model_name}:{prompt_templates}" + ) + return key, provider_config + def load_model( self, model_name: str, @@ -54,20 +159,41 @@ def load_model( model_name (str): Name of the model to load. prompt_template (Template): Prompt formatting template (used for non-instruct models). """ - key = ( - f"{model_name}:instruct" - if "instruct" in model_name.lower() - else f"{model_name}:{prompt_templates}" + key, provider_config = self._model_cache_key( + model_name, prompt_templates, model_parameters ) + if key is None: + return if key in self.__models: logging.info(f"Model {key} is already loaded, skipping loading process.") return try: - logging.info( - f"Loading model with cache directory: {self.__config.model_cache_dir}" - ) + if provider_config is not None: + # Provider-backed row: served over HTTP, no local ML stack needed. + if provider_config.resolved_kind(model_name) == "chat": + self.__models[key] = OpenAICompatibleChatModel( + model_name=model_name, config=provider_config + ) + else: + self.__models[key] = OpenAICompatibleCompletionModel( + model_name=model_name, + prompt_templates=json.loads(prompt_templates), + config=provider_config, + ) + logging.info(f"Provider model {key} is ready.") + return + + if torch is None: + raise RuntimeError( + "Local model loading requires the optional ML dependencies " + "(torch, transformers, and sentence-transformers)." + ) + from backend.completion.ChatCompletionModel import ChatCompletionModel + from backend.completion.TemplateCompletionModel import TemplateCompletionModel + + logging.info(f"Loading model with cache directory: {self.__config.model_cache_dir}") model_parameters = json.loads(model_parameters) if "instruct" in model_name.lower(): @@ -99,18 +225,21 @@ def load_model( del model_parameters[ "tokenizer" ] # Remove the tokenizer from the model parameters for better logging - logging.log( - logging.INFO, f"Model {key} is loaded successfully: {model_parameters}" - ) + logging.log(logging.INFO, f"Model {key} is loaded successfully: {model_parameters}") except Exception as e: logging.error(e) logging.error( f"Failed to load model '{model_name}' with prompt templates'{prompt_templates}' and model parameters'{model_parameters}'" ) - def get_model( - self, model_name: str, prompt_templates: str, model_parameters: str - ) -> Optional[Union[TemplateCompletionModel, ChatCompletionModel]]: + def get_model(self, model_name: str, prompt_templates: str, model_parameters: str) -> Optional[ + Union[ + TemplateCompletionModel, + ChatCompletionModel, + OpenAICompatibleChatModel, + OpenAICompatibleCompletionModel, + ] + ]: """ Retrieves a model instance. Loads and caches it if not already loaded. @@ -119,13 +248,11 @@ def get_model( prompt_template (Template): Prompt formatting template. Returns: - Optional[Union[TemplateCompletionModel, ChatCompletionModel]]: The model instance if loaded successfully. + Optional[Union[...]]: The model instance if loaded successfully. """ - key = ( - f"{model_name}:instruct" - if "instruct" in model_name.lower() - else f"{model_name}:{prompt_templates}" - ) + key, _ = self._model_cache_key(model_name, prompt_templates, model_parameters) + if key is None: + return None if key in self.__models: return self.__models[key] diff --git a/src/backend/redis_manager.py b/src/backend/redis_manager.py index ee9474d..8da4da5 100644 --- a/src/backend/redis_manager.py +++ b/src/backend/redis_manager.py @@ -11,6 +11,7 @@ import database.crud as crud import Queries from backend.utils import recursive_json_loads +from privacy.collection import lock_context_storage_allowed class RedisManager: @@ -19,6 +20,15 @@ class RedisManager: It manages auth_token -> { user_id, session_token } pairs and ensures session lifecycle via Redis expiration. """ + # Token types whose payload names the owning account in ``user_id``. + USER_SCOPED_TOKEN_TYPES = ( + "auth_token", + "acp_grant", + "acp_session", + "email_verification", + "password_reset", + ) + def __init__( self, host: str, @@ -29,9 +39,16 @@ def __init__( reset_password_token_expires_in_seconds: int = 900, token_hook_activation_in_seconds: int = 60, store_multi_file_context_on_db: bool = True, + password: Optional[str] = None, ): - # Initialize Redis client with given host and port - self.__redis_client = Redis(host=host, port=port, decode_responses=True) + # Initialize Redis client with given host and port (and AUTH when the + # deployment sets `requirepass`, see redis.prod.conf). + self.__redis_client = Redis( + host=host, + port=port, + password=password or None, + decode_responses=True, + ) self.session_token_expires_in_seconds = session_token_expires_in_seconds self.auth_token_expires_in_seconds = auth_token_expires_in_seconds self.email_verification_token_expires_in_seconds = ( @@ -52,6 +69,13 @@ def __init__( "Could not connect to Redis server. Check your configuration." ) + def ping(self) -> bool: + """Round-trip to Redis; ``False`` when it is unreachable (health checks).""" + try: + return bool(self.__redis_client.ping()) + except Exception: # noqa: BLE001 - a health probe reports, never raises + return False + def __get_exp(self, type: str) -> int: """ Get expiration time in seconds for different token types. @@ -62,6 +86,16 @@ def __get_exp(self, type: str) -> int: return self.auth_token_expires_in_seconds elif type == "session_token": return self.session_token_expires_in_seconds + elif type == "acp_grant": + # One-time launch grant handed to a locally-spawned agent process. + # Short-lived on purpose: it only has to survive the gap between the + # plugin writing the handoff file and the agent starting up. + return 300 + elif type == "acp_pending_grant": + return 300 + elif type == "acp_session": + # The agent process's working credential, refreshed on every use. + return 3600 elif type == "project_token": return -1 # project tokens do not expire by default elif type == "email_verification": @@ -102,9 +136,36 @@ def set(self, type: str, token: str, info: dict, force_reset_exp: bool = False): expiration - self.token_hook_activation_in_seconds, "", ) + if type == "session_token": + self.__extend_account_link(token, info) else: self.__redis_client.set(key, json_info, keepttl=True) + def touch(self, type: str, token: str) -> bool: + """Restart a token's expiry (and its hook's) without rewriting its value, + so a concurrent update is never lost; False when the token is gone.""" + if not token: + return False + expiration = self.__get_exp(type) + # A token whose hook is gone is ending (its expiry is being handled, or + # was missed): never extend it. + if self.__get_set_hook(type) and not self.__redis_client.expire( + f"{type}_hook:{token}", expiration - self.token_hook_activation_in_seconds + ): + return False + return bool(self.__redis_client.expire(f"{type}:{token}", expiration)) + + def __extend_account_link(self, session_token: str, session_info) -> None: + """Keep the account's link to a session alive as long as the session. + + Project and agent calls find the session through that link only, so a + link expiring first cut off a session still in use. + """ + user_id = session_info.get("user_token") if isinstance(session_info, dict) else None + link = self.get("user_token", user_id) if user_id else None + if link and link.get("session_token") == session_token: + self.__redis_client.expire(f"user_token:{user_id}", self.__get_exp("user_token")) + def get(self, type: str, token: str, reset_exp: bool = False) -> Optional[dict]: """ Retrieve token info from Redis and optionally reset its expiration. @@ -126,9 +187,31 @@ def get(self, type: str, token: str, reset_exp: bool = False) -> Optional[dict]: expiration - self.token_hook_activation_in_seconds, "", ) + if type == "session_token": + self.__extend_account_link(token, recursive_json_loads(data)) return recursive_json_loads(data) # Parse JSON string to dict return None + def consume(self, type: str, token: str) -> Optional[dict]: + """Atomically retrieve and invalidate a one-time token. + + Used for ACP launch grants, where a replayable token would let anyone + who reads the handoff file mint an agent session. GETDEL makes the + exchange single-use even under concurrent attempts. + """ + if not token: + return None + data = self.__redis_client.getdel(f"{type}:{token}") + if data: + return recursive_json_loads(data) + return None + + def discard(self, type: str, token: str) -> None: + """Remove an ephemeral token without the DB-persistence side effects of + ``delete`` (which tears down whole session graphs).""" + if token: + self.__redis_client.delete(f"{type}:{token}") + def delete(self, type: str, token: str, db_session: Session): """ Delete token and related data from Redis and persist relevant info to DB. @@ -167,18 +250,27 @@ def delete(self, type: str, token: str, db_session: Session): self.__redis_client.delete(f"{type}_hook:{token}") if session_dict: - # Remove user token if exists + # Remove the account's link to this session, unless the account + # has moved on to a newer session meanwhile: an old session that + # expires must not cut off the one the plugin is using now. user_token = session_dict.get("user_token") - if user_token: + user_info = self.get("user_token", user_token) if user_token else None + if user_info and user_info.get("session_token") == token: self.__redis_client.delete(f"user_token:{user_token}") - # Remove this session token from related project tokens + # Remove this session token from related project tokens, and any + # session already gone: one whose expiry nobody heard (no listener + # during a restart) must not keep the project open for good, and + # a second worker handling this same expiry finds it removed. for project_token in session_dict.get("project_tokens", []): project_dict = self.get("project_token", project_token) if project_dict: - new_session_tokens = project_dict.get("session_tokens", []) - new_session_tokens.remove(token) - project_dict["session_tokens"] = new_session_tokens + project_dict["session_tokens"] = [ + other + for other in project_dict.get("session_tokens", []) + if other != token + and self.__redis_client.exists(f"session_token:{other}") + ] self.set("project_token", project_token, project_dict) self.delete("project_token", project_token, db_session) @@ -187,7 +279,10 @@ def delete(self, type: str, token: str, db_session: Session): if project_dict: # Only proceed if no active session tokens remain for project token if len(project_dict.get("session_tokens", [])) == 0: - if self.store_multi_file_context_on_db: + # A context erased during this session is never written back. + if self.store_multi_file_context_on_db and not self.__redis_client.exists( + f"context_erased:{token}" + ): multi_file_contexts = project_dict.get( "multi_file_contexts", {} ) @@ -195,16 +290,15 @@ def delete(self, type: str, token: str, db_session: Session): "multi_file_context_changes", {} ) - # Verify all users allow storing context before persisting + # Verify all users allow storing context (and have not + # opted out of data collection) before persisting. The + # locked reads keep a concurrent erase from clearing the + # project before this write lands. project_users = crud.get_project_users(db_session, token) - allowed_to_store_context = True - for user_project in project_users: - user = crud.get_user_by_id(db_session, user_project.user_id) - if user and not json.loads(user.preference).get( - "store_context", False - ): - allowed_to_store_context = False - break + allowed_to_store_context = all( + lock_context_storage_allowed(db_session, user_project.user_id) + for user_project in project_users + ) if allowed_to_store_context: crud.update_project( db_session, @@ -214,12 +308,68 @@ def delete(self, type: str, token: str, db_session: Session): multi_file_context_changes=multi_file_context_changes, ), ) - # Delete project token from Redis - self.__redis_client.delete(key) + # Delete project token (and any erase marker) from Redis + self.__redis_client.delete(key, f"context_erased:{token}") else: # For other token types, just delete the key self.__redis_client.delete(key) + def mark_context_erased(self, user_id: str) -> None: + """ + Keep the account's live project context from being flushed to the database. + An erase clears the stored context, but the live session goes on serving + from Redis; without this marker that (pre-erase) context would be written + back when the session ends, even if the account opts back in meanwhile. + """ + user_info = self.get("user_token", user_id) or {} + session_info = self.get("session_token", user_info.get("session_token")) or {} + for project_token in session_info.get("project_tokens", []): + # A key of its own: the project entry is rewritten on every context + # update, which would race a read-modify-write marker. + self.__redis_client.set(f"context_erased:{project_token}", "1") + + def revoke_user_tokens(self, user_id: str) -> None: + """ + Invalidate every credential and session of an account, persisting nothing. + Used once the account is deleted: unlike delete(), which flushes session and + project state to the database, this only removes keys. + """ + for type in self.USER_SCOPED_TOKEN_TYPES: + for key in self.__redis_client.scan_iter(match=f"{type}:*"): + data = self.__redis_client.get(key) + info = recursive_json_loads(data) if data else None + if isinstance(info, dict) and str(info.get("user_id")) == user_id: + token = key.split(":", 1)[1] + self.__redis_client.delete(key, f"{type}_hook:{token}") + + user_info = self.get("user_token", user_id) + self.__redis_client.delete(f"user_token:{user_id}") + session_token = (user_info or {}).get("session_token") + if not session_token: + return + session_info = self.get("session_token", session_token) or {} + self.__redis_client.delete( + f"session_token:{session_token}", f"session_token_hook:{session_token}" + ) + # Detach the session from its projects; a project left without sessions + # is dropped without flushing its context to the database. + for project_token in session_info.get("project_tokens", []): + project_info = self.get("project_token", project_token) + if not project_info: + continue + remaining = [ + token + for token in project_info.get("session_tokens", []) + if token != session_token + ] + if remaining: + project_info["session_tokens"] = remaining + self.set("project_token", project_token, project_info) + else: + self.__redis_client.delete( + f"project_token:{project_token}", f"context_erased:{project_token}" + ) + def listen_for_expired_keys(self, session_factory): """ Listen for Redis key expiration events on token hooks and delete expired tokens accordingly. @@ -274,22 +424,3 @@ def close(self): self.__redis_client.close() except Exception: pass - - def cleanup(self, db_session: Session): - """ - Clean all tokens from Redis and persist necessary data to DB. - WARNING: Use with caution as it flushes the entire Redis DB. - """ - patterns = [ - "session_token:*", - "project_token:*", - "auth_token:*", - "user_token:*", - ] - for pattern in patterns: - # Iterate over all keys matching pattern and delete each - for key in self.__redis_client.keys(pattern): # type: ignore - type, token = key.split(":") - self.delete(type, token, db_session) - self.__redis_client.flushdb() - self.__redis_client.close() diff --git a/src/backend/resources/documentation/auth_workflows.puml b/src/backend/resources/documentation/auth_workflows.puml index f08c424..761ec69 100644 --- a/src/backend/resources/documentation/auth_workflows.puml +++ b/src/backend/resources/documentation/auth_workflows.puml @@ -5,9 +5,7 @@ actor User participant "Login/Signup UI" as UI participant "Auth Component" as Auth participant "API Utils" as API -participant "Auth Utils" as AuthUtils participant "Backend Server" as Server -participant "Google OAuth" as Google title Authentication Workflows @@ -23,64 +21,4 @@ group Regular Login UI --> User: Show authenticated UI end -' Google OAuth Login Flow (Existing User) -group Google OAuth Login (Existing User) - User -> UI: Click Google Login - UI -> Google: Initiate OAuth flow - Google --> UI: Return credential response - UI -> Auth: handleGoogleUser(googleUser) - Auth -> AuthUtils: handleGoogleAuth(googleUser) - AuthUtils -> AuthUtils: initGoogleOAuth(credentialResponse) - AuthUtils -> API: checkUserExists(email) - API -> Server: POST /user/exists - Server --> API: User exists = true - API --> AuthUtils: User exists - AuthUtils -> API: authenticateWithOAuth(provider, token) - API -> Server: POST /user/authenticate - Server --> API: Return user data & session token - API --> AuthUtils: Return authentication response - AuthUtils --> Auth: Call onExistingUser callback - Auth -> Auth: handleLogin(user) - Auth --> UI: Update UI with authenticated state - UI --> User: Show authenticated UI -end - -' Google OAuth Signup Flow (New User) -group Google OAuth Signup (New User) - User -> UI: Click Google Login - UI -> Google: Initiate OAuth flow - Google --> UI: Return credential response - UI -> Auth: handleGoogleUser(googleUser) - Auth -> AuthUtils: handleGoogleAuth(googleUser) - AuthUtils -> AuthUtils: initGoogleOAuth(credentialResponse) - AuthUtils -> API: checkUserExists(email) - API -> Server: POST /user/exists - Server --> API: User exists = false - API --> AuthUtils: User does not exist - AuthUtils --> Auth: Call onNewUser callback - Auth -> Auth: Show password creation modal - User -> Auth: Enter password - Auth -> Auth: handlePasswordSubmit(password) - Auth -> AuthUtils: createUser(userData) - AuthUtils -> API: POST /user/new - API -> Server: Create new user with Google credential - Server --> API: Return user data & session token - API --> AuthUtils: Return user creation response - AuthUtils --> Auth: Return user data - Auth --> UI: Update UI with authenticated state - UI --> User: Show authenticated UI -end - -' Regular Signup Flow -group Regular Signup - User -> UI: Enter name, email, password - UI -> Auth: handleSignup(userData) - Auth -> API: createUser(userData) - API -> Server: POST /user/new - Server --> API: Return user data & session token - API --> Auth: Return user creation response - Auth --> UI: Update UI with authenticated state - UI --> User: Show authenticated UI -end - -@enduml \ No newline at end of file +@enduml diff --git a/src/backend/routers/__init__.py b/src/backend/routers/__init__.py index 142ef62..f13eb17 100644 --- a/src/backend/routers/__init__.py +++ b/src/backend/routers/__init__.py @@ -1,6 +1,15 @@ -from fastapi import APIRouter +import logging + +from fastapi import APIRouter, Depends +from sqlalchemy import text + +from App import App +from backend.Responses import JsonResponseWithStatus # Import sub-routers for different parts of the application +from .acp import router as acp_router +from .agent import router as agent_router +from .agents import router as agent_tasks_router from .chat import router as chat_router from .completion import router as completion_router from .project import router as project_router @@ -9,6 +18,7 @@ from .ws import router as ws_routers from .analytics import router as analytics_router from .config import router as config_router +from .research import router as research_router # Main API router to aggregate and expose sub-routes router = APIRouter() @@ -34,9 +44,24 @@ # Include WebSocket-based endpoints (e.g., completions, chat) router.include_router(ws_routers, prefix="/ws", tags=["WebSocket"]) -# Include analytics and monitoring endpoints +# Include analytics and monitoring endpoints router.include_router(analytics_router, prefix="/analytics", tags=["Analytics"]) +# Agent subsystem. Both routers share the /agent prefix but authenticate +# differently: `agent_router` covers profile CRUD plus the readiness registry +# (admin/authenticated-user auth) and self-report ingestion + memory (ACP bearer +# auth from the locally launched agent process), while `agent_tasks_router` +# covers task lifecycle and the inference relay (plugin session cookie). +router.include_router(agent_router, prefix="/agent", tags=["Agent"]) +router.include_router(agent_tasks_router, prefix="/agent", tags=["Agent"]) + +# ACP grant handoff, so a locally launched agent process can authenticate. +router.include_router(acp_router, prefix="/acp", tags=["ACP"]) + +# Research platform (Issue 01+): ACP capability receipts and the compatibility +# gate. Admin-only; mounted under /api/research. +router.include_router(research_router, prefix="/research", tags=["Research"]) + @router.api_route("/ping", methods=["HEAD"]) def ping(): @@ -47,3 +72,32 @@ def ping(): dict: A simple response indicating the service is available. """ return {"Status": "Ok"} + + +@router.api_route("/health", methods=["GET", "HEAD"]) +def health(app: App = Depends(App.get_instance)): + """Dependency-aware health: the database and Redis must answer. + + `/api/ping` only proves the process is up; a study server whose database + is down would still pass the container health check while every telemetry + batch is refused. This route is what the production compose polls. + """ + checks: dict[str, str] = {} + db = app.get_db_session() + try: + db.execute(text("SELECT 1")) + checks["database"] = "ok" + except Exception as error: # noqa: BLE001 - reported, never raised + logging.warning("health: database check failed: %s", error) + checks["database"] = "error" + finally: + try: + db.close() + except Exception: # noqa: BLE001 + pass + checks["redis"] = "ok" if app.get_redis_manager().ping() else "error" + healthy = all(value == "ok" for value in checks.values()) + return JsonResponseWithStatus( + status_code=200 if healthy else 503, + content={"status": "ok" if healthy else "degraded", "checks": checks}, + ) diff --git a/src/backend/routers/acp/__init__.py b/src/backend/routers/acp/__init__.py new file mode 100644 index 0000000..e866234 --- /dev/null +++ b/src/backend/routers/acp/__init__.py @@ -0,0 +1,1546 @@ +"""ACP authorization and runtime configuration for locally launched agents. + +Endpoints (mounted under ``/api/acp``): + + POST /grant plugin mints a one-time launch grant + POST /session/exchange agent trades the grant for a session token + POST /session/validate-or-refresh agent revalidates and slides its expiry + GET /agent-config agent fetches its assigned runtime config + POST /persistent-auth-token admin mints a non-expiring auth token + +Why this exists: the built-in ``code4me2-agent`` runtime is a separate OS +process started by the IDE plugin. It has no session cookie, so it can't +authenticate the way the plugin does. The grant handoff gives it a credential of +its own, scoped to exactly one user, project and workspace directory. + +``/agent-config`` is where merge decision 6 lands for the built-in runtime: the +agent no longer reads a hardcoded local default, it asks the backend which +provider endpoint, model, tools and step budget to use. That's what makes the +assigned A/B profile actually govern the runtime we control, the same way the +inference relay governs the ones we don't. +""" + +from __future__ import annotations + +import json +import logging +import os +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +import httpx +from alembic.config import Config +from alembic.script import ScriptDirectory +from fastapi import APIRouter, Body, Cookie, Depends, Header, HTTPException, Query, Response +from fastapi.responses import JSONResponse +from pydantic import BaseModel, Field +from sqlalchemy import text +from sqlalchemy.exc import IntegrityError + +import Queries # noqa: TC001 - FastAPI evaluates route annotations at runtime +from agents import provider as provider_module +from agents import registry +from agents.tools import ( + CODE4ME2_AGENT_TOOLS, + command_key, + is_bare_command_name, + validate_command_timeout_seconds, + validate_commands_allowlist, + validate_commands_denylist, + validate_harness_options, +) +from App import App +from backend.acp_authorization import ( + AcpAuthorizationDenied, + AcpAuthorizationService, + AcpServerAuthorization, +) +from backend.Responses import ( + AcpAgentConfigGetResponse, + AcpAuthorizationError, + ExchangeAcpGrantPostResponse, + JsonResponseWithStatus, + PersistentAuthTokenPostResponse, + PrepareAcpGrantPostResponse, + ValidateAcpSessionPostResponse, +) +from backend.routers.agent.acp_auth import require_acp_scope, require_acp_server_scope +from backend.routers.agent.consent import resolve_store_agent_content_for_acp +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, + require_admin, +) +from backend.routers.research import access +from database import crud +from privacy import collection +from research.budget.meter import InferenceMeter +from research.runtime.sessions import store as session_store +from research.study.agents.enums import MANAGED_RUNTIME_FRAMEWORK +from utils import create_uuid + +router = APIRouter() + + +def _require_funded_access(db, *, account_id, study_id, enrollment_id=None): + """Re-check live enrollment/window/kill switch for a study-funded operation. + + The single funded gate for ACP paths (``access.require_funded_access``): + the account must have an ACTIVE enrollment, the study must not be + ``STUDY_STOPPED`` and must still be open, and no operator kill switch may be + engaged — at study scope or for that very enrollment. A previously issued + run/capability never bypasses current server state. Returns the live + enrollment row so callers can meter the call against its budget. + ``enrollment_id`` is accepted for call-site compatibility; the switch is + always scoped to the enrollment the gate resolves. + """ + try: + return access.require_funded_access( + db, account_id=account_id, study_id=study_id + ) + except access.FundedAccessRefused as exc: + raise HTTPException( + status_code=403, + detail={"code": exc.code, "message": exc.message}, + ) from exc + + +def _require_funded_task(db, task): + """Re-check live enrollment/window/kill switch for a study-funded task. + + Applied to the managed run replay and inference paths so a previously issued + run/capability never bypasses current server state. Returns the live + enrollment row, or ``None`` for a non-research task. + """ + if getattr(task, "study_id", None) is None: + return None + return _require_funded_access( + db, + account_id=getattr(task, "owner_user_id", None), + study_id=task.study_id, + enrollment_id=getattr(task, "enrollment_id", None), + ) + +# Fallback runtime config, used only when no agent profile can be resolved for +# the user (e.g. every profile has been deactivated). Deliberately minimal and +# read-only-ish: a handful of harmless inspection tools and no write tools, so +# an unconfigured agent degrades to something safe rather than to something +# powerful. +# +# Commands: current runtimes read ``commands_denylist`` (the programs the agent +# may not run) and run anything else. Runtime releases 0.0.6 and earlier only +# understand ``commands_allowlist`` and refuse a policy without it, so it is +# still sent, computed as before (a legacy snapshot allowlist, else the user's +# config-row allowlist, else this list) minus any blocked command. +FALLBACK_COMMANDS_ALLOWLIST = ["pwd", "ls", "cat", "grep", "rg"] +FALLBACK_TOOLS = ["read_file", "list_files", "search_files"] +FALLBACK_MODEL = "qwen2.5-coder:7b" +FALLBACK_MAX_ITERATIONS = 6 +FALLBACK_MAX_CONTEXT_TOKENS = 32_000 +MANAGED_PROTOCOL_VERSION = "1" +MANAGED_RUNTIME = MANAGED_RUNTIME_FRAMEWORK +SUPPORTED_APPROVAL_POLICIES = frozenset({"auto", "per_step", "suggestion_only"}) + +_BEARER_PREFIX = "Bearer " + +# The managed-agent schema readiness check compares the applied Alembic revision +# against the deployment's code head. Derive that head from the migration graph +# instead of baking a literal into the release: a hardcoded value silently goes +# stale the moment a migration is added and would falsely report readiness. +_REPO_ROOT = Path(__file__).resolve().parents[4] +_ALEMBIC_INI = _REPO_ROOT / "alembic.ini" +_ALEMBIC_SCRIPT_LOCATION = _REPO_ROOT / "src" / "database" / "migration" + +_expected_schema_revision_cache: Optional[str] = None +_expected_schema_revision_resolved = False + + +@router.post("/chat/completions") +async def acp_chat_completions( + body: dict = Body(...), + scope=Depends(require_acp_scope), + app: App = Depends(App.get_instance), +) -> Response: + """Proxy a built-in agent turn without exposing provider credentials locally.""" + if body.get("stream"): + raise HTTPException(status_code=400, detail="Streaming is not supported for ACP inference.") + + db = app.get_db_session() + try: + user_uuid = uuid.UUID(scope.user_id) + assignment = registry.resolve_assignment_context(db, user_uuid) + if assignment is None: + raise HTTPException(status_code=503, detail="No active agent profile is configured.") + # Live enrollment/window/kill-switch gate BEFORE provider resolution: a + # stopped or closed study never reaches a provider connection. + enrollment = _require_funded_access( + db, account_id=user_uuid, study_id=assignment.study_id + ) + profile = assignment.profile + funding_owner_user_id, owner_is_admin = provider_module.funding_owner_for_profile( + db, profile + ) + try: + connection = provider_module.resolve_task_connection( + db, + profile, + funding_owner_user_id, + owner_is_admin=owner_is_admin, + ) + upstream = provider_module.resolve_upstream( + model=profile.model, + connection=connection, + framework_version=profile.framework_version, + ) + except provider_module.ProviderReadinessError as exc: + raise HTTPException( + status_code=503, detail={"code": exc.code, "message": exc.message} + ) from exc + finally: + db.close() + + payload = dict(body) + payload["model"] = profile.model + if profile.temperature is not None: + payload["temperature"] = profile.temperature + # Research budgets: reserve the worst-case cost before the call, settle the + # actual usage afterwards (or void/forfeit the hold, see the meter). + meter = InferenceMeter( + app=app, + enrollment_id=enrollment.enrollment_id, + study_id=assignment.study_id, + connection_id=connection.connection_id, + model=profile.model, + entry_point="acp_chat_completions", + ) + refusal = await meter.reserve( + payload, request_id=str(uuid.uuid4()), upstream_base_url=upstream.base_url + ) + if refusal is not None: + return refusal + try: + async with httpx.AsyncClient(timeout=120) as client: + upstream_response = await client.post( + upstream.endpoint(responses_api=False), + json=payload, + headers=provider_module.upstream_request_headers( + upstream, session_key=scope.acp_token + ), + ) + except httpx.HTTPError as error: + await meter.resolve_transport_error(error) + return JSONResponse( + { + "error": { + "message": ( + "The model provider could not be reached " + f"({type(error).__name__}). Try again shortly." + ), + "type": "upstream_unavailable", + "code": 502, + } + }, + status_code=502, + ) + if upstream_response.status_code >= 400: + await meter.resolve_upstream_error(upstream_response.status_code) + else: + try: + upstream_json = upstream_response.json() + except ValueError: + upstream_json = None + await meter.resolve_response( + upstream_json, upstream_status=upstream_response.status_code + ) + return Response( + content=upstream_response.content, + status_code=upstream_response.status_code, + media_type=upstream_response.headers.get("content-type", "application/json"), + headers=meter.response_headers(), + ) + + +class ManagedRunRequest(BaseModel): + run_id: str = Field(min_length=1, max_length=200) + session_id: str = Field(min_length=1, max_length=500) + # Explicit canonical research attribution (ISSUE-02). The runtime sends the + # ids it received through the research handoff; the server validates them + # against the authorized account's active enrollment/session and never + # trusts them directly. + enrollment_id: Optional[uuid.UUID] = None + research_session_id: Optional[uuid.UUID] = None + + +class ManagedInferenceRequest(ManagedRunRequest): + request: dict + + +def _bearer_token(authorization: str) -> Optional[str]: + if not authorization.startswith(_BEARER_PREFIX): + return None + token = authorization[len(_BEARER_PREFIX) :] + return token or None + + +def _schema_revision(app: App) -> Optional[str]: + """Return the applied Alembic revision without relying on ORM metadata.""" + db = app.get_db_session() + try: + return db.execute(text("SELECT version_num FROM alembic_version")).scalar() + except Exception as error: + logging.warning("[ACP/capabilities] schema revision unavailable: %s", error) + return None + finally: + db.close() + + +def _derive_expected_schema_revision() -> Optional[str]: + """Return the single Alembic head, or ``None`` when it is not unique. + + ``alembic.ini`` records ``script_location``/``version_locations`` relative to + the package root, so resolve them explicitly: Alembic otherwise resolves + relative locations against the process working directory, which differs + between a dev shell and the deployed server. + """ + try: + cfg = Config(str(_ALEMBIC_INI)) + cfg.set_main_option("path_separator", "os") + cfg.set_main_option("script_location", str(_ALEMBIC_SCRIPT_LOCATION)) + cfg.set_main_option( + "version_locations", str(_ALEMBIC_SCRIPT_LOCATION / "versions") + ) + heads = ScriptDirectory.from_config(cfg).get_heads() + except Exception as error: + logging.warning( + "[ACP/capabilities] could not derive the Alembic head: %s", error + ) + return None + if len(heads) != 1: + logging.warning( + "[ACP/capabilities] expected exactly one Alembic head, found %d: %r", + len(heads), + heads, + ) + return None + return heads[0] + + +def expected_schema_revision() -> Optional[str]: + """Alembic revision the managed-agent schema must be at to be ready. + + ``MANAGED_SCHEMA_REQUIRED_REVISION`` is an operator escape hatch that pins an + explicit revision (for example in a staged rollout). Otherwise the expected + revision is the migration graph's single head; zero or multiple heads yield + ``None`` so the readiness probe fails closed instead of guessing. The derived + value is memoized because the head cannot change while the process runs. + """ + override = os.environ.get("MANAGED_SCHEMA_REQUIRED_REVISION", "").strip() + if override: + return override + global _expected_schema_revision_cache, _expected_schema_revision_resolved + if not _expected_schema_revision_resolved: + _expected_schema_revision_cache = _derive_expected_schema_revision() + _expected_schema_revision_resolved = True + return _expected_schema_revision_cache + + +@router.get("/capabilities") +def get_acp_capabilities(app: App = Depends(App.get_instance)) -> dict: + """Advertise the managed-agent contract and whether its schema is deployed. + + Deployment readiness is gated on the expected Alembic revision so a + readiness probe can fail before participant traffic is enabled: when the + schema is missing or behind, this returns HTTP 503 instead of a 200 with + ``schema_ready=False``. + """ + revision = _schema_revision(app) + expected = expected_schema_revision() + ready = revision is not None and revision == expected + payload = { + "managed_protocol_versions": [MANAGED_PROTOCOL_VERSION], + "supported_runtimes": [MANAGED_RUNTIME], + "schema_ready": ready, + "schema_revision": revision, + "expected_schema_revision": expected, + } + if not ready: + raise HTTPException( + status_code=503, + detail=( + f"Managed agent schema not ready: have {revision!r}, " + f"want {expected!r}" + ), + ) + return payload + + +@router.post( + "/persistent-auth-token", + response_model=PersistentAuthTokenPostResponse, + status_code=201, +) +def create_persistent_auth_token( + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +) -> JsonResponseWithStatus: + """Mint a non-expiring auth token for the calling admin. + + Needed for unattended runs (benchmark harnesses, long-lived dev setups) + where nobody is around to re-login when the normal token expires. Admin-only + and scoped to the caller's own user, since it deliberately sidesteps + session expiry. + """ + auth_token = create_uuid() + app.get_redis_manager().set( + "auth_token", + auth_token, + {"user_id": str(current_user.user_id)}, + ) + logging.info( + f"[ACP] persistent auth token minted for admin " + f"{str(current_user.user_id)[:8]}…" + ) + return JsonResponseWithStatus( + status_code=201, + content=PersistentAuthTokenPostResponse(auth_token=auth_token), + ) + + +@router.post("/grant", response_model=PrepareAcpGrantPostResponse, status_code=201) +def prepare_acp_grant( + request: Queries.PrepareAcpGrant, + app: App = Depends(App.get_instance), + auth_token: str = Cookie(""), +) -> JsonResponseWithStatus: + """Mint a one-time launch grant for the authenticated plugin. + + The plugin writes the returned grant where the agent process will find it. + Grants are short-lived and single-use, so a stale handoff file is inert. + + The grant is the participant's funded launch credential, so it is gated by + the same live-enrollment/window/kill-switch rule as every other funded path. + """ + user_id = _resolve_cookie_account(app, auth_token) + if user_id is not None: + db = app.get_db_session() + try: + # Study- and enrollment-scoped kill switches both refuse the grant. + access.require_funded_access(db, account_id=user_id) + except access.FundedAccessRefused as exc: + raise HTTPException( + status_code=403, + detail={"code": exc.code, "message": exc.message}, + ) from exc + finally: + db.close() + + service = AcpAuthorizationService(app.get_redis_manager()) + try: + prepared = service.prepare_grant( + auth_token=auth_token, + project_id=str(request.project_id), + workspace=request.workspace, + launch_id=request.launch_id, + path_format=request.path_format, + managed_protocol_version=request.managed_protocol_version, + ) + return JsonResponseWithStatus( + status_code=201, + content=PrepareAcpGrantPostResponse( + grant=prepared.grant, + workspace=prepared.workspace, + expires_in_seconds=prepared.expires_in_seconds, + launch_id=prepared.launch_id, + path_format=prepared.path_format, + managed_protocol_version=prepared.managed_protocol_version, + ), + ) + except AcpAuthorizationDenied as error: + return JsonResponseWithStatus( + status_code=401, + content=AcpAuthorizationError(message=str(error)), + ) + + +def _resolve_cookie_account(app: App, auth_token: str) -> Optional[uuid.UUID]: + """Resolve the cookie's account id, or ``None`` when it is not authenticated.""" + if not auth_token: + return None + try: + info = app.get_redis_manager().get("auth_token", auth_token) + if not isinstance(info, dict) or not info.get("user_id"): + return None + return uuid.UUID(str(info["user_id"])) + except Exception: + return None + + +@router.post("/session/exchange", response_model=ExchangeAcpGrantPostResponse) +def exchange_acp_grant( + request: Queries.ExchangeAcpGrant, + app: App = Depends(App.get_instance), +) -> JsonResponseWithStatus: + """Trade a launch grant for the agent process's session token. + + Unauthenticated by design — possession of the grant *is* the credential. + That's safe because the grant is single-use, short-lived, and bound to a + scope the redeemer cannot influence. + """ + service = AcpAuthorizationService(app.get_redis_manager()) + try: + session = service.exchange_grant(request.grant) + return JsonResponseWithStatus( + status_code=200, + content=ExchangeAcpGrantPostResponse( + acp_token=session.acp_token, + project_id=session.project_id, + workspace=session.workspace, + expires_in_seconds=session.expires_in_seconds, + ), + ) + except AcpAuthorizationDenied as error: + return JsonResponseWithStatus( + status_code=401, + content=AcpAuthorizationError(message=str(error)), + ) + + +@router.post( + "/session/validate-or-refresh", + response_model=ValidateAcpSessionPostResponse, +) +def validate_or_refresh_acp_session( + app: App = Depends(App.get_instance), + authorization: str = Header(default=""), +) -> JsonResponseWithStatus: + """Revalidate an agent session and slide its expiry. + + Also re-checks that the parent plugin login is still alive, so logging out + in the IDE invalidates the agent process rather than leaving it running with + a credential valid for another hour. + """ + token = _bearer_token(authorization) + if token is None: + return JsonResponseWithStatus( + status_code=401, content=AcpAuthorizationError() + ) + service = AcpAuthorizationService(app.get_redis_manager()) + try: + session = service.validate_or_refresh(token) + return JsonResponseWithStatus( + status_code=200, + content=ValidateAcpSessionPostResponse( + project_id=session.project_id, + workspace=session.workspace, + expires_in_seconds=session.expires_in_seconds, + ), + ) + except AcpAuthorizationDenied as error: + return JsonResponseWithStatus( + status_code=401, + content=AcpAuthorizationError(message=str(error)), + ) + + +@router.get("/agent-config", response_model=AcpAgentConfigGetResponse) +def get_acp_agent_config( + app: App = Depends(App.get_instance), + authorization: str = Header(default=""), + managed_protocol_version: Optional[str] = Query(default=None), +) -> JsonResponseWithStatus: + """Return the runtime config for the calling agent process. + + Resolution order: + + 1. the user's assigned agent profile (the sticky A/B draw) — this is the + normal path, and it's what makes the assignment govern the runtime; + 2. the user's ``config`` row, whose optional ``agent`` section can block + more commands (``commands_denylist``, added to the profile's list: an + operational safety setting rather than an experimental condition, so it + stays outside the profile). Its legacy ``commands_allowlist`` still + narrows or replaces the allowlist sent to runtimes 0.0.6 and earlier, + from which blocked commands are removed; + 3. the conservative fallback constants above. + + ``command_timeout_seconds`` and ``harness_options`` are returned only when + the profile sets them, so a config without them is unchanged. + + No API key is ever returned — only ``api_key_ref``, the environment + variable name the agent reads it from locally. + """ + token = _bearer_token(authorization) + if token is None: + return JsonResponseWithStatus( + status_code=401, content=AcpAuthorizationError() + ) + + service = AcpAuthorizationService(app.get_redis_manager()) + try: + session = service.validate_or_refresh(token) + except AcpAuthorizationDenied: + return JsonResponseWithStatus( + status_code=401, content=AcpAuthorizationError() + ) + + # Fallback defaults, overridden below where a profile/config supplies values. + # Managed callers never use these fallbacks (see managed branch below); + # they fail closed instead so a malformed study profile cannot start work. + commands_allowlist = list(FALLBACK_COMMANDS_ALLOWLIST) + tools = list(FALLBACK_TOOLS) + tools_parse_failed = False + model = FALLBACK_MODEL + max_iterations = FALLBACK_MAX_ITERATIONS + agent_profile: Optional[str] = None + framework_version: Optional[str] = None + base_url: Optional[str] = None + api_key_ref: Optional[str] = None + approval_policy: Optional[str] = None + temperature: Optional[float] = None + max_context_tokens: Optional[int] = None + system_prompt: Optional[str] = None + profile_allowlist: Optional[list[str]] = None + profile_denylist: Optional[list[str]] = None + commands_denylist: list[str] = [] + command_timeout_seconds: Optional[int] = None + harness_options: Optional[dict] = None + store_agent_content = False + # Set when the blocked commands cannot be known (a malformed setting a + # legacy caller is served around): run_command is then withheld, because + # dropping a denylist would widen what the agent may run. + withhold_commands = False + + db = app.get_db_session() + try: + try: + user_uuid = uuid.UUID(str(session.user_id)) + except (ValueError, TypeError): + logging.warning( + f"[ACP/agent-config] scope user_id is not a UUID: {session.user_id!r}" + ) + user_uuid = None + + if user_uuid is not None: + # Import locally to avoid a circular import at module load time + # (agents.registry → database.crud → … → this router package). + from agents import registry + + assignment = registry.resolve_assignment_context(db, user_uuid) + profile = assignment.profile if assignment is not None else None + if profile is not None: + agent_profile = profile.name + framework_version = profile.framework_version + model = profile.model + approval_policy = profile.approval_policy + temperature = profile.temperature + max_context_tokens = profile.max_context_tokens + system_prompt = getattr(profile, "system_prompt", None) + max_iterations = max(1, int(profile.max_steps or max_iterations)) + try: + parsed_tools = json.loads(profile.tools_json or "[]") + if isinstance(parsed_tools, list): + tools = [str(t) for t in parsed_tools] + else: + tools_parse_failed = True + except (json.JSONDecodeError, TypeError): + tools_parse_failed = True + logging.warning( + f"[ACP/agent-config] profile {profile.name!r} has invalid " + f"tools_json — using fallback tool set" + ) + try: + ( + profile_allowlist, + profile_denylist, + command_timeout_seconds, + harness_options, + ) = _profile_harness_settings(profile) + except ValueError: + if managed_protocol_version is not None: + # Fail closed (503 below) rather than run the arm + # without its frozen command/harness settings. + raise + logging.warning( + f"[ACP/agent-config] profile {profile.name!r} has invalid " + f"command/harness settings — ignoring them" + ) + # Keep the arm's blocked commands when they are valid on + # their own; otherwise no command may run. + raw_denylist = getattr(profile, "commands_denylist", None) + try: + if raw_denylist is not None: + profile_denylist = validate_commands_denylist(raw_denylist) + except ValueError: + withhold_commands = True + logging.info( + f"[ACP/agent-config] profile={profile.name!r} " + f"runtime={framework_version} model={model} " + f"tools={len(tools)} max_iterations={max_iterations}" + ) + else: + logging.warning( + "[ACP/agent-config] no active profile for user — " + "serving conservative fallback config" + ) + + store_agent_content = resolve_store_agent_content_for_acp( + db, + session.user_id, + study_id=assignment.study_id if assignment is not None else None, + ) + + # Blocked commands are an operational guardrail, so the config row + # can add to them per user, independently of the profile. + config_allowlist: Optional[list[str]] = None + config_denylist: Optional[list[str]] = None + user = crud.get_user_by_id(db, user_uuid) if user_uuid else None + if user is not None: + config_row = crud.get_config_by_id(db, user.config_id) + if config_row is not None and config_row.config_data: + overrides = _parse_agent_config_section(config_row.config_data) + raw_allowlist = overrides.get("commands_allowlist") + if isinstance(raw_allowlist, list): + config_allowlist = [ + str(c).strip() for c in raw_allowlist if c + ] + try: + config_denylist = _config_row_denylist(overrides) + except ValueError: + if managed_protocol_version is not None: + raise + logging.warning( + "[ACP/agent-config] the user's config row has an invalid " + "commands_denylist — withholding run_command" + ) + withhold_commands = True + commands_denylist = _effective_commands_denylist( + profile_denylist, config_denylist + ) + commands_allowlist = _effective_commands_allowlist( + profile_allowlist, config_allowlist, blocked=commands_denylist + ) + except Exception as error: + if managed_protocol_version is not None: + raise HTTPException( + status_code=503, + detail="Assigned managed-agent policy could not be resolved", + ) from error + # Legacy/developer runtimes retain their conservative fallback behavior. + logging.error( + f"[ACP/agent-config] config resolution failed, using fallbacks — {error}", + exc_info=True, + ) + # The arm's blocked commands may not have been read. + withhold_commands = True + finally: + db.close() + if withhold_commands: + tools = [tool for tool in tools if str(tool).strip() != "run_command"] + + transport: Optional[str] = None + if managed_protocol_version is not None: + if managed_protocol_version != MANAGED_PROTOCOL_VERSION: + raise HTTPException(status_code=426, detail="Unsupported managed protocol") + if agent_profile is None: + raise HTTPException( + status_code=503, + detail="No active study agent profile is assigned to this user", + ) + if tools_parse_failed: + raise HTTPException( + status_code=503, detail="Assigned profile has invalid tools" + ) + if ( + not model + or approval_policy not in SUPPORTED_APPROVAL_POLICIES + or max_iterations < 1 + or not set(tools).issubset(CODE4ME2_AGENT_TOOLS) + ): + raise HTTPException( + status_code=503, detail="Assigned profile policy is incomplete" + ) + if framework_version != MANAGED_RUNTIME: + raise HTTPException( + status_code=409, + detail=f"Assigned runtime {framework_version!r} is not managed in this release", + ) + # Managed runtimes always use this server as transport. Provider location + # and credential references stay server-side. + base_url = None + api_key_ref = None + if max_context_tokens is None: + max_context_tokens = FALLBACK_MAX_CONTEXT_TOKENS + transport = "managed_backend" + + return JsonResponseWithStatus( + status_code=200, + content=AcpAgentConfigGetResponse( + agent_profile=agent_profile, + framework_version=framework_version, + model=model, + base_url=base_url, + api_key_ref=api_key_ref, + commands_allowlist=commands_allowlist, + commands_denylist=commands_denylist, + tools=tools, + max_iterations=max_iterations, + max_context_tokens=max_context_tokens, + approval_policy=approval_policy, + temperature=temperature, + system_prompt=system_prompt, + command_timeout_seconds=command_timeout_seconds, + harness_options=harness_options, + store_agent_content=store_agent_content, + transport=transport, + managed_protocol_version=( + MANAGED_PROTOCOL_VERSION if transport is not None else None + ), + ), + ) + + +class _InvalidHarnessSetting(ValueError): + """A frozen command/harness setting is malformed; ``detail`` is the 503 text.""" + + def __init__(self, detail: str): + super().__init__(detail) + self.detail = detail + + +def _profile_harness_settings( + profile, +) -> tuple[Optional[list[str]], Optional[list[str]], Optional[int], Optional[dict]]: + """The profile's built-in runtime command/harness settings (decision D-01). + + Returns ``(legacy commands_allowlist, commands_denylist, + command_timeout_seconds, harness_options)``, each ``None`` when unset (only + snapshots frozen before the denylist carry an allowlist). Raises + :class:`_InvalidHarnessSetting` (a ``ValueError``) when a frozen value is + malformed, so callers fail closed instead of dropping an arm's setting. + Only shapes are checked here: whether the verify command runs a blocked + program is refused when the profile is saved and frozen, and at run time the + runtime skips such a command. A later change to the matching rule must not + take a running arm down. + """ + allowlist = getattr(profile, "commands_allowlist", None) + denylist = getattr(profile, "commands_denylist", None) + timeout = getattr(profile, "command_timeout_seconds", None) + options = getattr(profile, "harness_options", None) + try: + if allowlist is not None: + allowlist = validate_commands_allowlist(allowlist) + if denylist is not None: + denylist = validate_commands_denylist(denylist) + except ValueError as error: + raise _InvalidHarnessSetting("Assigned command policy is invalid") from error + try: + if timeout is not None: + timeout = validate_command_timeout_seconds(timeout) + except ValueError as error: + raise _InvalidHarnessSetting("Assigned profile command timeout is invalid") from error + try: + if options is not None: + options = validate_harness_options(options) + except ValueError as error: + raise _InvalidHarnessSetting("Assigned profile harness options are invalid") from error + return allowlist, denylist, timeout, options + + +def _effective_commands_allowlist( + profile_allowlist: Optional[list[str]], + config_allowlist: Optional[list[str]], + *, + blocked: list[str], +) -> list[str]: + """The legacy command allowlist runtimes 0.0.6 and earlier receive. + + A (legacy snapshot) profile allowlist is the arm's condition: the user's + config row can only narrow it (intersection, profile order kept). Without + one the config row replaces the fallback list. Blocked commands are removed + either way, so an old runtime never runs one. + """ + if profile_allowlist is None: + if config_allowlist is not None: + allowlist = list(config_allowlist) + else: + allowlist = list(FALLBACK_COMMANDS_ALLOWLIST) + elif config_allowlist is None: + allowlist = list(profile_allowlist) + else: + permitted = set(config_allowlist) + allowlist = [command for command in profile_allowlist if command in permitted] + blocked_keys = {command_key(command) for command in blocked} + return [command for command in allowlist if command_key(command) not in blocked_keys] + + +def _config_row_denylist(overrides: dict) -> Optional[list[str]]: + """The extra commands a user's config row blocks; ``ValueError`` when malformed. + + Unlike the legacy allowlist, a malformed list is never ignored: dropping it + would widen what the agent may run. + """ + raw = overrides.get("commands_denylist") + if raw is None: + return None + if not isinstance(raw, list) or not all( + isinstance(command, str) and is_bare_command_name(command.strip()) for command in raw + ): + raise ValueError("the config row's commands_denylist must list bare command names") + return [command.strip() for command in raw] + + +def _effective_commands_denylist( + profile_denylist: Optional[list[str]], config_denylist: Optional[list[str]] +) -> list[str]: + """The blocked commands a runtime receives. + + The profile's list is the arm's condition; the user's config row can only + add to it (profile order first, then the row's other names, ignoring case). + """ + blocked = list(profile_denylist or []) + seen = {command.lower() for command in blocked} + for command in config_denylist or []: + if command.lower() not in seen: + seen.add(command.lower()) + blocked.append(command) + return blocked + + +def _managed_policy(db, user_id: uuid.UUID, profile, *, study_id: Optional[uuid.UUID] = None) -> dict: + """Build the executable v1 policy; malformed study profiles fail closed.""" + if profile.framework_version != MANAGED_RUNTIME: + raise HTTPException( + status_code=409, + detail=f"Assigned runtime {profile.framework_version!r} is not managed in this release", + ) + try: + tools = json.loads(profile.tools_json) + except (TypeError, json.JSONDecodeError) as error: + raise HTTPException(status_code=503, detail="Assigned profile has invalid tools") from error + if ( + not isinstance(tools, list) + or not all(isinstance(item, str) and item.strip() for item in tools) + or not set(tools).issubset(CODE4ME2_AGENT_TOOLS) + ): + raise HTTPException(status_code=503, detail="Assigned profile has invalid tools") + try: + max_iterations = int(profile.max_steps) + except (TypeError, ValueError): + max_iterations = 0 + temperature = profile.temperature + max_context_tokens = profile.max_context_tokens + if isinstance(temperature, bool) or ( + temperature is not None + and ( + not isinstance(temperature, (int, float)) + or not 0.0 <= float(temperature) <= 2.0 + ) + ): + raise HTTPException(status_code=503, detail="Assigned profile temperature is invalid") + if max_context_tokens is None: + max_context_tokens = FALLBACK_MAX_CONTEXT_TOKENS + if ( + isinstance(max_context_tokens, bool) + or not isinstance(max_context_tokens, int) + or max_context_tokens < 1 + ): + raise HTTPException(status_code=503, detail="Assigned profile context limit is invalid") + if ( + not isinstance(profile.model, str) + or not profile.model.strip() + or profile.approval_policy not in SUPPORTED_APPROVAL_POLICIES + or max_iterations < 1 + ): + raise HTTPException(status_code=503, detail="Assigned profile policy is incomplete") + try: + profile_allowlist, profile_denylist, command_timeout_seconds, harness_options = ( + _profile_harness_settings(profile) + ) + except _InvalidHarnessSetting as error: + raise HTTPException(status_code=503, detail=error.detail) from error + + config_allowlist: Optional[list[str]] = None + config_denylist: Optional[list[str]] = None + user = crud.get_user_by_id(db, user_id) + if user is not None: + config_row = crud.get_config_by_id(db, user.config_id) + if config_row is not None and config_row.config_data: + overrides = _parse_agent_config_section(config_row.config_data) + raw_allowlist = overrides.get("commands_allowlist") + if isinstance(raw_allowlist, list): + if not all(isinstance(command, str) and command.strip() for command in raw_allowlist): + raise HTTPException(status_code=503, detail="Assigned command policy is invalid") + config_allowlist = [command.strip() for command in raw_allowlist] + try: + config_denylist = _config_row_denylist(overrides) + except ValueError as error: + raise HTTPException( + status_code=503, detail="Assigned command policy is invalid" + ) from error + commands_denylist = _effective_commands_denylist(profile_denylist, config_denylist) + commands_allowlist = _effective_commands_allowlist( + profile_allowlist, config_allowlist, blocked=commands_denylist + ) + + policy = { + "version": MANAGED_PROTOCOL_VERSION, + "transport": "managed_backend", + "agent_profile": profile.name, + "framework_version": profile.framework_version, + "model": profile.model.strip(), + "tools": tools, + "approval_policy": profile.approval_policy, + "temperature": float(temperature) if temperature is not None else None, + "max_iterations": max_iterations, + "max_context_tokens": max_context_tokens, + # Legacy: only runtimes 0.0.6 and earlier read the allowlist; current + # ones run any program except ``commands_denylist``. + "commands_allowlist": commands_allowlist, + "commands_denylist": commands_denylist, + "store_agent_content": resolve_store_agent_content_for_acp( + db, str(user_id), study_id=study_id + ), + } + # The managed runtime re-applies this run policy before every prompt, so + # the frozen prompt travels with it (and is recorded on the task). A + # prompt-less arm keeps exactly the policy it had before the field existed. + system_prompt = getattr(profile, "system_prompt", None) + if system_prompt is not None: + policy["system_prompt"] = system_prompt + # Likewise the command timeout and harness switches (decision D-01): an arm + # without them keeps a byte-identical policy. + if command_timeout_seconds is not None: + policy["command_timeout_seconds"] = command_timeout_seconds + if harness_options is not None: + policy["harness_options"] = harness_options + return policy + + +@router.get("/readiness") +def get_participant_readiness( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +) -> dict: + """Resolve and validate the signed-in participant's sticky study arm.""" + db = app.get_db_session() + try: + from agents import registry + + resolution = registry.resolve_assignment_context(db, current_user.user_id) + if resolution is None: + raise HTTPException( + status_code=503, + detail="No active study agent profile is assigned to this user", + ) + profile = resolution.profile + policy = _managed_policy( + db, current_user.user_id, profile, study_id=resolution.study_id + ) + funding_owner_user_id, owner_is_admin = provider_module.funding_owner_for_profile( + db, profile + ) + try: + connection = provider_module.resolve_task_connection( + db, + profile, + funding_owner_user_id, + owner_is_admin=owner_is_admin, + ) + provider_module.resolve_upstream( + model=profile.model, + connection=connection, + framework_version=profile.framework_version, + ) + except provider_module.ProviderReadinessError as exc: + raise HTTPException( + status_code=503, + detail={"code": exc.code, "message": exc.message}, + ) from exc + return { + "ready": True, + "managed_protocol_version": MANAGED_PROTOCOL_VERSION, + "runtime": profile.framework_version, + "agent_profile": profile.name, + "policy_version": policy["version"], + } + finally: + db.close() + + +def _scope_ids(scope: AcpServerAuthorization) -> tuple[uuid.UUID, uuid.UUID, uuid.UUID]: + try: + return ( + uuid.UUID(str(scope.user_id)), + uuid.UUID(str(scope.project_id)), + uuid.UUID(str(scope.session_id)), + ) + except (TypeError, ValueError) as error: + raise HTTPException(status_code=401, detail="ACP authorization scope is malformed") from error + + +def _is_external_run_id_unique_violation(error: Exception) -> bool: + """True only for the raced ``agent_task.external_run_id`` unique insert. + + The default deployment dialect is psycopg2, where a raced concurrent + insert surfaces as ``IntegrityError`` with ``orig.pgcode == "23505"`` and + ``orig.diag.constraint_name`` naming the ``external_run_id`` constraint. + Every other failure — unrelated integrity violations, post-commit errors + such as a refresh failure, generic exceptions — must take the logged 500 + path and never replay. + """ + if not isinstance(error, IntegrityError): + return False + orig = getattr(error, "orig", None) + if getattr(orig, "pgcode", None) != "23505": + return False + diag = getattr(orig, "diag", None) + constraint_name = getattr(diag, "constraint_name", None) + return isinstance(constraint_name, str) and "external_run_id" in constraint_name + + +def _require_managed_task(db, body: ManagedRunRequest, scope: AcpServerAuthorization): + user_id, project_id, _ = _scope_ids(scope) + task = crud.get_agent_task_by_external_run_id(db, body.run_id) + if task is None: + raise HTTPException(status_code=404, detail="Managed run does not exist") + if ( + task.owner_user_id != user_id + or task.owner_project_id != project_id + or task.agent_session_id != body.session_id + or task.source != "code4me2_agent" + ): + raise HTTPException(status_code=409, detail="Managed run belongs to another scope") + return task + + +def _live_session_of_window(db, research_session_id, enrollment_id): + """The live research session of the IDE window ``research_session_id`` belonged to. + + The plugin writes its window's research session into the agent's ACP entry, + and an idle rotation keeps that entry, so the agent goes on naming the ended + session. The window's live session continues it. A revoked session, or one + of another enrollment, is returned unchanged and refused by the caller. + """ + row = session_store.get_session(db, research_session_id) + if ( + row is None + or row.enrollment_id != enrollment_id + or str(getattr(row, "state", "")) != "ended" + ): + return research_session_id + live = session_store.get_active_session_for_context(db, enrollment_id, row.context_id) + return live.session_id if live is not None else research_session_id + + +@router.post("/runs") +def create_managed_run( + body: ManagedRunRequest, + app: App = Depends(App.get_instance), + scope: AcpServerAuthorization = Depends(require_acp_server_scope), +) -> JSONResponse: + """Idempotently create a self-reporting managed runtime task.""" + user_id, project_id, parent_session_id = _scope_ids(scope) + db = app.get_db_session() + try: + existing = crud.get_agent_task_by_external_run_id(db, body.run_id) + if existing is not None: + task = _require_managed_task(db, body, scope) + # A replayed run must still satisfy current funded state. + _require_funded_task(db, task) + if not isinstance(task.policy_snapshot, dict): + raise HTTPException(status_code=503, detail="Managed run policy is unavailable") + return JSONResponse( + {"task_id": str(task.task_id), "run_id": body.run_id, "policy": task.policy_snapshot}, + status_code=200, + ) + + from agents import registry + + assignment = registry.resolve_assignment_context(db, user_id) + if assignment is None: + raise HTTPException( + status_code=503, detail="No active study agent profile is assigned to this user" + ) + profile = assignment.profile + # Serialize with a concurrent erase: from here to the task insert nothing + # commits (the assignment lookup above may, on first use), so a task either + # commits before the erase (and is erased with the rest) or waits for it and + # then sees the opt-out. + collection.lock_account(db, user_id) + if assignment.study_id is None: + raise HTTPException( + status_code=409, + detail={ + "code": "RESEARCH_CONTEXT_REQUIRED", + "message": "a managed run requires an active research enrollment", + }, + ) + # A newly created run must satisfy the live funded gate too, not only a + # replay: a stopped study, revoked enrollment or engaged kill switch is + # refused before a task exists (ISSUE-01/ISSUE-02). + _require_funded_access(db, account_id=user_id, study_id=assignment.study_id) + binding = access.resolve_research_binding( + db, account_id=user_id, study_id=assignment.study_id + ) + if binding is None: + raise HTTPException( + status_code=409, + detail={ + "code": "RESEARCH_CONTEXT_REQUIRED", + "message": "no active enrollment for the assigned study", + }, + ) + if ( + body.enrollment_id is not None + and body.enrollment_id != binding.enrollment_id + ): + raise HTTPException( + status_code=409, + detail={ + "code": "RESEARCH_CONTEXT_MISMATCH", + "message": "the supplied enrollment does not match the active context", + }, + ) + research_session_id = body.research_session_id + if research_session_id is not None: + research_session_id = _live_session_of_window( + db, research_session_id, binding.enrollment_id + ) + if ( + binding.research_session_id is not None + and binding.research_session_id != research_session_id + ): + raise HTTPException( + status_code=409, + detail={ + "code": "RESEARCH_CONTEXT_MISMATCH", + "message": "the supplied research session is not the active context", + }, + ) + session_row = session_store.get_session(db, research_session_id) + terminal_states = {"ended", "revoked"} + if ( + session_row is None + or session_row.enrollment_id != binding.enrollment_id + or str(getattr(session_row, "state", "")) in terminal_states + ): + raise HTTPException( + status_code=409, + detail={ + "code": "RESEARCH_CONTEXT_MISMATCH", + "message": "the supplied research session is not active for this enrollment", + }, + ) + else: + research_session_id = binding.research_session_id + if research_session_id is None: + # Explicit context is required: zero or more than one active session + # must never be guessed from account-wide state. + raise HTTPException( + status_code=409, + detail={ + "code": "RESEARCH_CONTEXT_AMBIGUOUS", + "message": ( + "the account has no single active research session; " + "supply research_session_id" + ), + }, + ) + policy = _managed_policy( + db, user_id, profile, study_id=assignment.study_id + ) + task = crud.create_agent_task( + db, + agent_profile=profile.name, + model=profile.model, + approval_policy=profile.approval_policy, + tools_json=profile.tools_json, + temperature=profile.temperature, + framework_version=profile.framework_version, + session_id=parent_session_id, + owner_user_id=user_id, + funding_owner_user_id=getattr(profile, "funding_owner_user_id", None), + owner_project_id=project_id, + external_run_id=body.run_id, + agent_session_id=body.session_id, + source="code4me2_agent", + status="running", + started_at=datetime.now(timezone.utc), + policy_snapshot=policy, + study_id=assignment.study_id, + study_assignment_id=assignment.assignment_id, + profile_id=profile.profile_id, + consent_content_storage=policy["store_agent_content"], + research_session_id=research_session_id, + enrollment_id=binding.enrollment_id, + ) + return JSONResponse( + {"task_id": str(task.task_id), "run_id": body.run_id, "policy": policy}, + status_code=201, + ) + except HTTPException: + raise + except Exception as error: + if not _is_external_run_id_unique_violation(error): + # Only the raced unique insert may replay: any other failure + # (funding/policy refusals surface as HTTPException above; + # unrelated integrity violations, post-commit failures and generic + # errors land here) is a logged 500. A retry then replays + # idempotently through the pre-read path above. + db.rollback() + logging.error("[ACP/runs] failed to create managed run: %s", error, exc_info=True) + raise HTTPException(status_code=500, detail="Failed to create managed run") from error + # A concurrent identical request raced the unique external_run_id. + # Roll back before attempting an ownership-safe idempotent read. + db.rollback() + existing = crud.get_agent_task_by_external_run_id(db, body.run_id) + if existing is not None: + task = _require_managed_task(db, body, scope) + # A replayed run must still satisfy current funded state. + _require_funded_task(db, task) + if not isinstance(task.policy_snapshot, dict): + raise HTTPException(status_code=503, detail="Managed run policy is unavailable") + return JSONResponse( + {"task_id": str(task.task_id), "run_id": body.run_id, "policy": task.policy_snapshot}, + status_code=200, + ) + logging.error("[ACP/runs] failed to create managed run: %s", error, exc_info=True) + raise HTTPException(status_code=500, detail="Failed to create managed run") from error + finally: + db.close() + + +@router.post("/inference") +async def run_managed_inference( + body: ManagedInferenceRequest, + app: App = Depends(App.get_instance), + scope: AcpServerAuthorization = Depends(require_acp_server_scope), +): + """Relay a managed runtime model call without proxy-generated telemetry.""" + if body.request.get("stream"): + raise HTTPException(status_code=400, detail="Managed protocol v1 requires non-streaming inference") + if "input" in body.request or not isinstance(body.request.get("messages"), list): + raise HTTPException(status_code=400, detail="Managed protocol v1 requires a Chat Completions request") + + _, _, parent_session_id = _scope_ids(scope) + db = app.get_db_session() + try: + task = _require_managed_task(db, body, scope) + # Live enrollment/window/kill-switch gate for funded managed inference. + enrollment = _require_funded_task(db, task) + policy = task.policy_snapshot + if not _valid_managed_policy_snapshot(policy): + raise HTTPException(status_code=503, detail="Managed run policy is unavailable") + profile = None + if task.profile_id is not None: + profile = crud.get_agent_profile_by_id(db, task.profile_id) + if profile is None: + raise HTTPException(status_code=503, detail="Managed run provider configuration is unavailable") + funding_owner_user_id, owner_is_admin = provider_module.funding_owner_for_task( + db, task + ) + try: + connection = provider_module.resolve_task_connection( + db, profile, funding_owner_user_id, owner_is_admin=owner_is_admin + ) + except provider_module.ProviderReadinessError as exc: + raise HTTPException( + status_code=503, detail={"code": exc.code, "message": exc.message} + ) from exc + snapshot = { + "task_id": task.task_id, + "agent_profile": task.agent_profile, + "model": policy.get("model"), + "temperature": policy.get("temperature"), + "tools_json": json.dumps(policy.get("tools", [])), + "framework_version": task.framework_version, + "connection": connection, + "content_included": bool(policy.get("store_agent_content", False)), + "study_id": getattr(task, "study_id", None), + "enrollment_id": None if enrollment is None else enrollment.enrollment_id, + "research_session_id": getattr(task, "research_session_id", None), + } + finally: + db.close() + + from agents import inference + + # Research budgets: a study-funded managed call is metered against the + # participant's balance (reserve → forward → settle/void/forfeit). + meter = None + if snapshot["enrollment_id"] is not None: + meter = InferenceMeter( + app=app, + enrollment_id=snapshot["enrollment_id"], + study_id=snapshot["study_id"], + connection_id=snapshot["connection"].connection_id, + model=snapshot["model"], + entry_point="acp_inference", + research_session_id=snapshot["research_session_id"], + ) + + model_request = dict(body.request) + model_request["model"] = snapshot["model"] + _enforce_inference_tool_policy(model_request, policy_tools=policy["tools"]) + if snapshot["temperature"] is None: + # A participant-supplied request may not broaden or change the assigned + # experiment policy. Absence in the snapshot means provider default, + # not permission to provide a client-side override. + model_request.pop("temperature", None) + else: + model_request["temperature"] = snapshot["temperature"] + + return await inference.run_inference( + task_uuid=snapshot["task_id"], + session_uuid=parent_session_id, + openai_body=model_request, + enrichment=None, + agent_profile=snapshot["agent_profile"], + model=snapshot["model"], + connection=snapshot["connection"], + temperature=snapshot["temperature"], + framework_version=snapshot["framework_version"], + profile_tools_json=snapshot["tools_json"], + content_included=snapshot["content_included"], + record_observation_events=False, + meter=meter, + app=app, + ) + + +def _valid_harness_options_snapshot(options: object) -> bool: + """Whether a run policy's ``harness_options`` has a valid shape. + + The verify command is not re-checked against the policy's blocked commands: + the user's config row may block more, and the runtime then skips it. + """ + try: + validate_harness_options(options) + except ValueError: + return False + return True + + +def _valid_managed_policy_snapshot(policy: object) -> bool: + if not isinstance(policy, dict): + return False + temperature = policy.get("temperature") + context_limit = policy.get("max_context_tokens") + iterations = policy.get("max_iterations") + tools = policy.get("tools") + # Optional keys (decision D-01): absent (or null) in policies of arms that + # do not set them and in every policy written before they existed. + command_timeout = policy.get("command_timeout_seconds") + harness_options = policy.get("harness_options") + commands_denylist = policy.get("commands_denylist") + return bool( + policy.get("version") == MANAGED_PROTOCOL_VERSION + and isinstance(policy.get("model"), str) + and policy["model"].strip() + and policy.get("approval_policy") in SUPPORTED_APPROVAL_POLICIES + and isinstance(iterations, int) + and not isinstance(iterations, bool) + and iterations >= 1 + and isinstance(context_limit, int) + and not isinstance(context_limit, bool) + and context_limit >= 1 + and ( + temperature is None + or ( + isinstance(temperature, (int, float)) + and not isinstance(temperature, bool) + and 0.0 <= float(temperature) <= 2.0 + ) + ) + and isinstance(tools, list) + and all(isinstance(tool, str) and tool.strip() for tool in tools) + and set(tools).issubset(CODE4ME2_AGENT_TOOLS) + and (command_timeout is None or _valid_command_timeout(command_timeout)) + and (harness_options is None or _valid_harness_options_snapshot(harness_options)) + and ( + commands_denylist is None + or ( + isinstance(commands_denylist, list) + and all( + isinstance(command, str) and command.strip() + for command in commands_denylist + ) + ) + ) + ) + + +def _valid_command_timeout(value: object) -> bool: + try: + validate_command_timeout_seconds(value) + except ValueError: + return False + return True + + +def _enforce_inference_tool_policy( + model_request: dict, + *, + policy_tools: list[str], +) -> None: + """Prevent a managed client from broadening its snapshotted tool set.""" + allowed = set(policy_tools) + wildcard_mcp = "mcp__*" in allowed + requested = model_request.get("tools") + if requested is None: + model_request.pop("tool_choice", None) + return + if not isinstance(requested, list): + raise HTTPException(status_code=400, detail="Managed inference tools must be a list") + for definition in requested: + if not isinstance(definition, dict): + raise HTTPException(status_code=400, detail="Managed inference tool definition is invalid") + function = definition.get("function") + name = function.get("name") if isinstance(function, dict) else None + if not isinstance(name, str) or not name.strip(): + raise HTTPException(status_code=400, detail="Managed inference tool definition is invalid") + if name not in allowed and not (wildcard_mcp and name.startswith("mcp__")): + raise HTTPException( + status_code=403, + detail=f"Tool {name!r} is not enabled by the managed run policy", + ) + tool_choice = model_request.get("tool_choice") + if isinstance(tool_choice, dict): + function = tool_choice.get("function") + selected_name = function.get("name") if isinstance(function, dict) else None + if not isinstance(selected_name, str) or not selected_name.strip(): + raise HTTPException(status_code=400, detail="Managed inference tool choice is invalid") + requested_names = { + definition["function"]["name"] + for definition in requested + } + if selected_name not in requested_names: + raise HTTPException( + status_code=403, + detail=f"Tool choice {selected_name!r} is not present in the managed request", + ) + elif tool_choice not in (None, "auto", "none", "required"): + raise HTTPException(status_code=400, detail="Managed inference tool choice is invalid") + if not requested: + model_request.pop("tools", None) + model_request.pop("tool_choice", None) + + +def _parse_agent_config_section(config_data: str) -> dict: + """Extract the optional ``agent`` section from a user's config blob. + + Configs are stored as JSON in this platform. Group-5's original version + also regex-parsed a HOCON ``agent { … }`` block; that is dropped here + because regex-parsing a config language is fragile and the JSON path is the + only shape this repo actually writes. A non-JSON config simply yields no + overrides rather than being half-parsed. + """ + try: + parsed = json.loads(config_data) + except (json.JSONDecodeError, ValueError, TypeError): + return {} + if not isinstance(parsed, dict): + return {} + agent_section = parsed.get("agent") + return agent_section if isinstance(agent_section, dict) else {} diff --git a/src/backend/routers/agent/__init__.py b/src/backend/routers/agent/__init__.py new file mode 100644 index 0000000..48ee4d0 --- /dev/null +++ b/src/backend/routers/agent/__init__.py @@ -0,0 +1,22 @@ +"""Agent sub-router: profiles/readiness, telemetry ingestion, memory. + +Mounted at ``/api/agent`` alongside ``backend.routers.agents`` (task lifecycle +and the inference relay), which shares the same prefix. The split is by +*authentication*, not by URL space: + +* this package's ``ingest`` and ``memory`` routers authenticate the locally + launched agent process via its ACP bearer token; +* ``profiles`` is admin/authenticated-user via ``auth_utils``; +* ``backend.routers.agents`` authenticates the IDE plugin via its session cookie. +""" + +from fastapi import APIRouter + +from .ingest import router as ingest_router +from .memory import router as memory_router +from .profiles import router as profiles_router + +router = APIRouter() +router.include_router(profiles_router) +router.include_router(ingest_router) +router.include_router(memory_router) diff --git a/src/backend/routers/agent/acp_auth.py b/src/backend/routers/agent/acp_auth.py new file mode 100644 index 0000000..30b44d0 --- /dev/null +++ b/src/backend/routers/agent/acp_auth.py @@ -0,0 +1,66 @@ +"""Shared FastAPI dependency for ACP-bearer-authenticated agent endpoints. + +The built-in ``code4me2-agent`` runtime is a separate local process with no +session cookie, so it authenticates with the ``acp_session`` bearer token it got +from the grant exchange (see ``backend.acp_authorization``). This module turns +that header into a resolved scope for route handlers. + +Every failure mode collapses to one flat 401 so the response never reveals +*which* check failed. +""" + +from __future__ import annotations + +from typing import Optional + +from fastapi import Depends, Header, HTTPException + +from App import App +from backend.acp_authorization import ( + AcpServerAuthorization, + AcpSessionAuthorization, + authorize_acp_bearer, +) + +_UNAUTHORIZED_DETAIL = "ACP authorization is invalid or expired." + + +def require_acp_scope( + app: App = Depends(App.get_instance), + authorization: str = Header(default=""), +) -> AcpSessionAuthorization: + """Resolve ``Authorization: Bearer `` to a session scope. + + The returned scope's ``user_id`` / ``project_id`` / ``workspace`` are + server-derived from the original grant, so handlers can treat them as + trusted and must never take those values from the request body instead. + """ + scope = authorize_acp_bearer(app.get_redis_manager(), authorization) + if not isinstance(scope, AcpSessionAuthorization): + raise HTTPException(status_code=401, detail=_UNAUTHORIZED_DETAIL) + return scope + + +def require_acp_server_scope( + app: App = Depends(App.get_instance), + authorization: str = Header(default=""), +) -> AcpServerAuthorization: + """Same, but also resolves the project record. + + For handlers that need the project's stored context, not just its id. + """ + scope = authorize_acp_bearer( + app.get_redis_manager(), authorization, server_scope=True + ) + if not isinstance(scope, AcpServerAuthorization): + raise HTTPException(status_code=401, detail=_UNAUTHORIZED_DETAIL) + return scope + + +def optional_acp_scope( + app: App = Depends(App.get_instance), + authorization: str = Header(default=""), +) -> Optional[AcpSessionAuthorization]: + """Resolve the scope if present, else None — for endpoints valid either way.""" + scope = authorize_acp_bearer(app.get_redis_manager(), authorization) + return scope if isinstance(scope, AcpSessionAuthorization) else None diff --git a/src/backend/routers/agent/consent.py b/src/backend/routers/agent/consent.py new file mode 100644 index 0000000..45d0444 --- /dev/null +++ b/src/backend/routers/agent/consent.py @@ -0,0 +1,184 @@ +"""Server-side consent gate for persisting agent *content*. + +Merge decision 3: the mechanism is group-21's server-enforced double gate, with +the default flipped to ON. + +* **Server-enforced.** Whether message text, tool arguments/results, diffs and + raw payloads get written is decided here from server state — never from a flag + in the request body. A compromised or stale client cannot turn content capture + on, and cannot turn it off either (which matters for research integrity as + much as for privacy). +* **Account opt-out overrides everything.** An account that opted out of data + collection (:mod:`privacy.collection`) gets no content stored, research-bound + or not. +* **Study policy first (ISSUE-01).** For a research-bound context (an explicit + ``study_id``), permission comes from the active enrollment plus the study's + frozen telemetry policy, resolved by + :mod:`research.telemetry.content_policy`. The legacy account preference is + *not* study consent and is only consulted for genuinely non-research contexts + (no study binding), where there is no study policy to enforce. +* **Default ON for non-research rows.** When a user row predates the preference + key it is absent, and the non-research default is ``True``. Missing or + malformed *study* policy never defaults to True: it denies. + +Structural telemetry (token counts, latency, span tree, tool *names*, content +*lengths*) is never gated: it carries no user or code text, and it's what the +A/B analysis actually runs on. + +Every caller funnels through this module so there is exactly one place where the +decision is made. +""" + +from __future__ import annotations + +import json +import logging +import uuid +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + +from database import crud +from database.db_schemas import ( + STORE_AGENT_CONTENT_DEFAULT, + STORE_AGENT_CONTENT_KEY, +) +from privacy import collection +from research.telemetry.content_policy import ( + DENY_DATA_COLLECTION_OPTED_OUT, + DENY_NO_PARTICIPANT, + LEGACY_PREFERENCE, + ContentPolicyDecision, + resolve_study_content_policy, +) + + +def _preference_allows_content(preference: Optional[str]) -> bool: + """Read the flag out of a user's stored preference JSON. + + An absent key means the row predates the preference, so the documented + default applies. Malformed JSON is treated as absent rather than as a + refusal, because a corrupt preference blob shouldn't silently change what a + study collects — it should behave like an un-migrated row. + """ + if not preference: + return STORE_AGENT_CONTENT_DEFAULT + try: + parsed = json.loads(preference) + except (ValueError, TypeError): + logging.warning( + "[Agent/consent] unparseable user preference — " + f"falling back to default store_agent_content={STORE_AGENT_CONTENT_DEFAULT}" + ) + return STORE_AGENT_CONTENT_DEFAULT + if not isinstance(parsed, dict): + return STORE_AGENT_CONTENT_DEFAULT + return bool(parsed.get(STORE_AGENT_CONTENT_KEY, STORE_AGENT_CONTENT_DEFAULT)) + + +def _legacy_preference_decision( + db: Session, user_id: Optional[uuid.UUID] +) -> ContentPolicyDecision: + """The non-research decision: the account's own stored preference.""" + if user_id is None: + return ContentPolicyDecision(allowed=False, reason=DENY_NO_PARTICIPANT) + try: + user = crud.get_user_by_id(db, user_id) + except Exception as e: + logging.warning(f"[Agent/consent] user lookup failed for {user_id} — {e}") + return ContentPolicyDecision(allowed=False, reason=DENY_NO_PARTICIPANT) + if user is None: + return ContentPolicyDecision(allowed=False, reason=DENY_NO_PARTICIPANT) + return ContentPolicyDecision( + allowed=_preference_allows_content(user.preference), + reason=LEGACY_PREFERENCE, + ) + + +def resolve_content_policy_for_user( + db: Session, + user_id: Optional[uuid.UUID], + *, + study_id: Optional[uuid.UUID] = None, + enrollment_id: Optional[uuid.UUID] = None, +) -> ContentPolicyDecision: + """Resolve the content-storage decision for a known user id. + + An account that opted out of data collection is denied first, whatever the + study or preference says. Otherwise, with a ``study_id`` the study's frozen + policy is authoritative (and a missing/malformed policy denies). Without one, + the legacy account preference applies: an unattributable request gets the + conservative answer, while a real non-research account keeps its documented + default. + """ + if user_id is not None and collection.opted_out_at(db, user_id) is not None: + return ContentPolicyDecision(allowed=False, reason=DENY_DATA_COLLECTION_OPTED_OUT) + if study_id is not None: + return resolve_study_content_policy( + db, + account_id=user_id, + study_id=study_id, + enrollment_id=enrollment_id, + ) + return _legacy_preference_decision(db, user_id) + + +def resolve_store_agent_content_for_user( + db: Session, + user_id: Optional[uuid.UUID], + *, + study_id: Optional[uuid.UUID] = None, + enrollment_id: Optional[uuid.UUID] = None, +) -> bool: + """Boolean form of :func:`resolve_content_policy_for_user`.""" + return resolve_content_policy_for_user( + db, user_id, study_id=study_id, enrollment_id=enrollment_id + ).allowed + + +def resolve_store_agent_content( + db: Session, + session_id: uuid.UUID, + *, + study_id: Optional[uuid.UUID] = None, + enrollment_id: Optional[uuid.UUID] = None, +) -> bool: + """Resolve the preference for the user owning ``session_id``. + + This is the plugin/proxy path, where the caller is identified by the + session cookie. + """ + try: + session = crud.get_session_by_id(db, session_id) + except Exception as e: + logging.warning(f"[Agent/consent] session lookup failed for {session_id} — {e}") + return False + if session is None or session.user_id is None: + return False + return resolve_store_agent_content_for_user( + db, session.user_id, study_id=study_id, enrollment_id=enrollment_id + ) + + +def resolve_store_agent_content_for_acp( + db: Session, + raw_user_id: str, + *, + study_id: Optional[uuid.UUID] = None, + enrollment_id: Optional[uuid.UUID] = None, +) -> bool: + """Resolve the preference for an ACP-authorized agent process. + + ``raw_user_id`` comes from the server-derived ACP scope (see + ``backend.acp_authorization``), so it is trustworthy — but it arrives as a + string and has to parse as a UUID to be usable. + """ + try: + user_uuid = uuid.UUID(str(raw_user_id)) + except (ValueError, TypeError, AttributeError): + logging.warning(f"[Agent/consent] ACP scope user_id is not a UUID: {raw_user_id!r}") + return False + return resolve_store_agent_content_for_user( + db, user_uuid, study_id=study_id, enrollment_id=enrollment_id + ) diff --git a/src/backend/routers/agent/ingest.py b/src/backend/routers/agent/ingest.py new file mode 100644 index 0000000..5e22125 --- /dev/null +++ b/src/backend/routers/agent/ingest.py @@ -0,0 +1,538 @@ +"""Self-report telemetry ingestion for the built-in code4me2-agent runtime. + +Endpoints (mounted under ``/api/agent``): + + POST /events/ingest ingest a batch of self-reported events + GET /runs/{run_id} read back one run and its events + +This is the ingestion half of merge decision 2. Where third-party agents are +*observed* by the proxy relay, the runtime we own *reports* — and both write +into the same ``agent_event`` table, so no downstream consumer has to know which +runtime produced a row. + +The runtime mints its own ``run_id`` before it ever contacts the backend, so the +first batch for a run lazily creates the backing ``agent_task``, drawing the +user's assigned A/B profile at that point (the same sticky draw the plugin path +uses). Subsequent batches attach to the existing task. +""" + +from __future__ import annotations + +import logging +import uuid +from datetime import datetime +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, Field + +from agents import ingest as ingest_module +from agents import lifecycle, registry +from App import App +from backend.acp_authorization import AcpSessionAuthorization # noqa: TC001 - FastAPI route +from backend.Responses import JsonResponseWithStatus +from backend.routers.agent.acp_auth import require_acp_scope +from backend.routers.agent.consent import resolve_store_agent_content_for_acp +from backend.routers.research.access import ( + FundedAccessRefused, + require_live_enrollment, + resolve_research_binding, +) +from database import crud +from privacy import collection +from research.participants import identity as identity_store +from research.participants.enums import EnrollmentStatus +from research.runtime.sessions import store as session_store +from research.telemetry.adapters import ( + CanonicalIngestionFailed, # noqa: TC001 - raised by agents.ingest +) + +router = APIRouter() + +# Terminal runtime statuses that should finalize the backing task. +_TERMINAL_RUN_STATUSES = frozenset({"completed", "failed", "cancelled", "error"}) + +# Research session states that can never accept new content. +_TERMINAL_SESSION_STATES = frozenset({"ended", "revoked"}) + +# Requested session and enrollment must agree with the persisted task binding. +_RESEARCH_CONTEXT_MISMATCH = { + "code": "RESEARCH_CONTEXT_MISMATCH", + "message": "the run's research context does not match the task", +} + + +class AgentRunEnvelope(BaseModel): + """Run-level metadata accompanying every batch. + + ``owner_user_id`` / ``owner_project_id`` are deliberately absent: ownership + comes from the server-derived ACP scope, so a runtime can't claim someone + else's run. + """ + + run_id: str = Field(..., min_length=1) + session_id: str = Field(..., min_length=1) + status: str = Field(..., min_length=1) + source: str = Field(default="code4me2_agent", min_length=1) + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + + +class AgentEventEnvelope(BaseModel): + """One self-reported event, in the runtime's own envelope format. + + Kept loose on purpose: ``payload`` / ``metrics`` / ``observability`` are + free-form because the runtime evolves faster than this schema, and + ``agents.ingest`` decides which keys are promoted to typed columns versus + routed into ``extra_json``. Rejecting unknown keys here would mean a + runtime change silently dropping telemetry. + """ + + event_id: str = Field(..., min_length=1) + run_id: str = Field(..., min_length=1) + schema_version: str = Field(..., min_length=1) + event_type: str = Field(..., min_length=1) + timestamp: datetime + sequence: int = Field(..., ge=1) + source: str = Field(..., min_length=1) + session_id: str = Field(..., min_length=1) + request_id: str = Field(..., min_length=1) + parent_event_id: Optional[str] = None + message_id: Optional[str] = None + payload: dict[str, Any] = Field(default_factory=dict) + raw_payload: Optional[dict[str, Any]] = None + metrics: dict[str, Any] = Field(default_factory=dict) + # Descriptive only — the storage decision is made server-side. Retained for + # audit so a client/server mismatch is visible. + privacy: dict[str, Any] = Field(default_factory=dict) + observability: dict[str, Any] = Field(default_factory=dict) + + +class AgentEventBatchIngest(BaseModel): + run: AgentRunEnvelope + events: list[AgentEventEnvelope] = Field(default_factory=list) + + +def _refuse_research_context(code: str, message: str) -> HTTPException: + return HTTPException(status_code=409, detail={"code": code, "message": message}) + + +def _validate_task_research_context(db, task) -> None: + """Fail closed when a task's research enrollment/session is terminal. + + A run that was created while the enrollment was active must not keep + submitting content after a revoke/stop/completed transition (ISSUE-01). + The response is a typed terminal reason, never a silent 200. + """ + enrollment_id = getattr(task, "enrollment_id", None) + research_session_id = getattr(task, "research_session_id", None) + if enrollment_id is None and research_session_id is None: + return + enrollment_row = ( + identity_store.get_enrollment(db, enrollment_id) + if enrollment_id is not None + else None + ) + if enrollment_row is None: + raise _refuse_research_context( + "RESEARCH_CONTEXT_UNAVAILABLE", + "the run's research enrollment no longer exists", + ) + if enrollment_row.status != EnrollmentStatus.ACTIVE.value: + raise _refuse_research_context( + "ENROLLMENT_NOT_ACTIVE", + "the run's research enrollment is no longer active", + ) + if getattr(task, "study_id", None) is not None: + try: + require_live_enrollment( + db, + account_id=getattr(task, "owner_user_id", None), + study_id=task.study_id, + ) + except FundedAccessRefused as exc: + raise _refuse_research_context(exc.code, exc.message) from exc + if research_session_id is not None: + session_row = session_store.get_session(db, research_session_id) + if session_row is None or str(getattr(session_row, "state", "")) in _TERMINAL_SESSION_STATES: + raise _refuse_research_context( + "SESSION_TERMINAL", + "the run's research session is no longer active", + ) + + +def _backfill_task_research_context(db, task, *, owner_user_uuid) -> None: + """Backfill missing canonical attribution from an unambiguous live context. + + Called only for an existing task: a legacy row created before attribution + landed must not silently drop canonical facts. A mismatch or an ambiguous + context is refused by :func:`_validate_task_research_context`. + """ + if getattr(task, "study_id", None) is None: + return + if task.enrollment_id is not None and task.research_session_id is not None: + return + binding = resolve_research_binding( + db, account_id=owner_user_uuid, study_id=task.study_id + ) + if binding is None: + raise _refuse_research_context( + "RESEARCH_CONTEXT_REQUIRED", + "the run has no active research enrollment for its study", + ) + changed = False + if task.enrollment_id is None: + task.enrollment_id = binding.enrollment_id + changed = True + if task.research_session_id is None and binding.research_session_id is not None: + task.research_session_id = binding.research_session_id + changed = True + if changed: + db.commit() + logging.info( + f"[Agent/ingest] backfilled research attribution for run " + f"{task.external_run_id} enrollment={str(task.enrollment_id)[:8]}…" + ) + + +def _resolve_or_create_task( + db, + *, + run: AgentRunEnvelope, + scope: AcpSessionAuthorization, +): + """Find the task backing this run, creating it on the first batch. + + Returns ``(task, created)``. Raises 403 if a task exists for this run id but + belongs to a different user or project, and 409 if it belongs to a different + ACP session — the run id is runtime-generated, so ownership checks are the + only thing preventing one agent process from writing into another scope's + task. + """ + task = crud.get_agent_task_by_external_run_id(db, run.run_id) + + try: + owner_user_uuid = uuid.UUID(str(scope.user_id)) + except (ValueError, TypeError): + raise HTTPException(status_code=401, detail="ACP scope has no valid user") + try: + owner_project_uuid = uuid.UUID(str(scope.project_id)) + except (ValueError, TypeError): + owner_project_uuid = None + + if task is not None: + if task.owner_user_id is not None and task.owner_user_id != owner_user_uuid: + logging.warning( + f"[Agent/ingest] 403 — run {run.run_id} owned by another user" + ) + raise HTTPException( + status_code=403, + detail="Agent run is not authorized for this ACP session.", + ) + # Managed runs are bound to one project and one ACP session at creation + # (/api/acp/runs). A second project reusing the same run_id — e.g. two + # IDE windows sharing a user — must not append telemetry to another + # project's task. + if ( + task.owner_project_id is not None + and owner_project_uuid is not None + and task.owner_project_id != owner_project_uuid + ): + logging.warning( + f"[Agent/ingest] 403 — run {run.run_id} owned by another project" + ) + raise HTTPException( + status_code=403, + detail="Agent run is not authorized for this ACP session.", + ) + if ( + task.agent_session_id is not None + and run.session_id != task.agent_session_id + ): + logging.warning( + f"[Agent/ingest] 409 — run {run.run_id} belongs to another ACP session" + ) + raise HTTPException( + status_code=409, + detail="Agent run belongs to another ACP session.", + ) + # ISSUE-02: validate (and backfill, when unambiguous) the canonical + # research attribution before accepting any new fact. A terminal + # enrollment/session refuses here instead of silently accepting content. + _backfill_task_research_context(db, task, owner_user_uuid=owner_user_uuid) + _validate_task_research_context(db, task) + return task, False + + assignment = registry.resolve_assignment_context(db, owner_user_uuid) + if assignment is None: + raise HTTPException( + status_code=503, + detail="No active agent profiles are configured on the server", + ) + profile = assignment.profile + # Serialize with a concurrent erase: from here to the task insert nothing + # commits (the assignment lookup above may, on first use), so a task either + # commits before the erase (and is erased with the rest) or waits for it and + # then sees the opt-out. + collection.lock_account(db, owner_user_uuid) + content_included = resolve_store_agent_content_for_acp( + db, scope.user_id, study_id=assignment.study_id + ) + binding = None + if assignment.study_id is not None: + # A study-bound self-report run must carry the canonical attribution + # (ISSUE-02). Never create an unattributed research task: ambiguity is + # refused, not guessed. + binding = resolve_research_binding( + db, account_id=owner_user_uuid, study_id=assignment.study_id + ) + if binding is None: + raise _refuse_research_context( + "RESEARCH_CONTEXT_REQUIRED", + "no active research enrollment exists for the assigned study", + ) + if binding.research_session_id is None: + raise _refuse_research_context( + "RESEARCH_CONTEXT_AMBIGUOUS", + "the account has no single active research session", + ) + task = crud.create_agent_task( + db, + agent_profile=profile.name, + model=profile.model, + approval_policy=profile.approval_policy, + tools_json=profile.tools_json, + temperature=profile.temperature, + framework_version=profile.framework_version, + source="code4me2_agent", + owner_user_id=owner_user_uuid, + funding_owner_user_id=getattr(profile, "funding_owner_user_id", None), + owner_project_id=owner_project_uuid, + external_run_id=run.run_id, + agent_session_id=run.session_id, + status="running", + started_at=run.started_at, + study_id=assignment.study_id, + study_assignment_id=assignment.assignment_id, + profile_id=profile.profile_id, + consent_content_storage=content_included, + # Explicit phase-05 attribution: resolved from the authorized account + # and the frozen assignment, never guessed. A non-research task keeps + # these NULL (an explicit "no research context"). + research_session_id=binding.research_session_id if binding else None, + enrollment_id=binding.enrollment_id if binding else None, + ) + logging.info( + f"[Agent/ingest] created task {task.task_id} for run {run.run_id} " + f"profile={profile.name!r} user={str(owner_user_uuid)[:8]}…" + ) + return task, True + + +@router.post("/events/ingest", summary="Ingest self-reported agent telemetry") +def ingest_agent_events( + body: AgentEventBatchIngest, + app: App = Depends(App.get_instance), + scope: AcpSessionAuthorization = Depends(require_acp_scope), +) -> JsonResponseWithStatus: + """Ingest a batch of events reported by the built-in runtime. + + Idempotent by the runtime's own event ids: a retried batch reports its + duplicates rather than double-writing, because the uploader can't tell + whether a timed-out POST was applied. + """ + if any(event.run_id != body.run.run_id for event in body.events): + raise HTTPException( + status_code=400, detail="All events must belong to the request run_id." + ) + if body.run.source != "code4me2_agent" or any( + event.source != "code4me2_agent" for event in body.events + ): + raise HTTPException( + status_code=400, + detail="Managed telemetry must use the code4me2_agent source.", + ) + if any(event.session_id != body.run.session_id for event in body.events): + raise HTTPException( + status_code=400, + detail="All events must belong to the request ACP session.", + ) + + db = app.get_db_session() + try: + task, _created = _resolve_or_create_task(db, run=body.run, scope=scope) + + # Consent is resolved from the *server's* record of the user's + # preference, not from the `privacy` block in the payload. The research + # enrollment gate is applied on top, so a non-ACTIVE research + # enrollment denies content collection even when the legacy preference + # would allow it. + content_included = resolve_store_agent_content_for_acp( + db, + scope.user_id, + study_id=task.study_id, + enrollment_id=task.enrollment_id, + ) + + ingested, skipped = ingest_module.ingest_event_batch( + db, + task_id=task.task_id, + events=[e.model_dump() for e in body.events], + content_included=content_included, + agent_profile=task.agent_profile, + ) + + # A run reporting a terminal status is finished, so aggregate it now + # rather than waiting for a close call the runtime may never make. + if body.run.status.strip().lower() in _TERMINAL_RUN_STATUSES: + reported = body.run.status.strip().lower() + status = ( + "failed" + if reported in ("failed", "error") + else "cancelled" + if reported == "cancelled" + else "done" + ) + lifecycle.finalize_agent_task(db, task.task_id, status=status) + + return JsonResponseWithStatus( + status_code=201, + content={ + "run_id": body.run.run_id, + "task_id": str(task.task_id), + "ingested_event_count": ingested, + "duplicate_event_ids": skipped, + "content_stored": content_included, + }, + ) + except HTTPException: + raise + except CanonicalIngestionFailed as error: + db.rollback() + # A research-bound canonical write failure must never be reported as a + # success or silently written to the legacy table (ISSUE-07). + status_code = 503 if error.retryable else 409 + raise HTTPException( + status_code=status_code, + detail={"code": error.reason, "message": error.message}, + ) from error + except Exception as error: + db.rollback() + logging.error(f"[Agent/ingest] error ingesting events: {error}", exc_info=True) + raise HTTPException( + status_code=500, detail="Server failed to ingest agent events." + ) + finally: + db.close() + + +@router.get("/runs/{run_id}", summary="Read back a self-reported agent run") +def get_agent_run( + run_id: str, + session_id: str = Query(..., min_length=1, max_length=500), + app: App = Depends(App.get_instance), + scope: AcpSessionAuthorization = Depends(require_acp_scope), +) -> JsonResponseWithStatus: + """Return one run's task row and its events, for the owning agent process.""" + db = app.get_db_session() + try: + task = crud.get_agent_task_by_external_run_id(db, run_id) + if task is None: + raise HTTPException(status_code=404, detail="Agent run not found.") + try: + owner_user_uuid = uuid.UUID(str(scope.user_id)) + except (ValueError, TypeError): + raise HTTPException(status_code=401, detail="ACP scope has no valid user") + if task.owner_user_id != owner_user_uuid: + raise HTTPException( + status_code=403, + detail="Agent run is not authorized for this ACP session.", + ) + try: + owner_project_uuid = uuid.UUID(str(scope.project_id)) + except (ValueError, TypeError): + raise HTTPException(status_code=401, detail="ACP scope has no valid project") + if task.owner_project_id != owner_project_uuid: + raise HTTPException( + status_code=403, + detail="Agent run is not authorized for this ACP project.", + ) + if task.agent_session_id != session_id: + raise HTTPException( + status_code=409, + detail="Agent run belongs to another ACP session.", + ) + + events = crud.get_agent_events_by_task(db, task.task_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "run": _serialize_task(task), + "events": [_serialize_event(e) for e in events], + }, + ) + except HTTPException: + raise + except Exception as error: + logging.error( + f"[Agent/ingest] error retrieving run {run_id}: {error}", exc_info=True + ) + raise HTTPException( + status_code=500, detail="Server failed to retrieve agent run." + ) + finally: + db.close() + + +def _iso(value: Any) -> Any: + return value.isoformat() if isinstance(value, datetime) else value + + +def _serialize_task(task) -> dict[str, Any]: + return { + "task_id": str(task.task_id), + "run_id": task.external_run_id, + "agent_session_id": task.agent_session_id, + "status": task.status, + "source": task.source, + "agent_profile": task.agent_profile, + "model": task.model, + "framework_version": task.framework_version, + "total_steps": task.total_steps, + "input_tokens": task.input_tokens, + "output_tokens": task.output_tokens, + "started_at": _iso(task.started_at), + "completed_at": _iso(task.completed_at), + "created_at": _iso(task.created_at), + } + + +def _serialize_event(event) -> dict[str, Any]: + """Serialise an event row. + + Content columns are echoed as stored — which means null when consent was + off. Nothing is reconstructed here, so this endpoint can't be used to read + back content the gate refused to persist. + """ + return { + "event_id": str(event.event_id), + "event_index": event.event_index, + "event_type": event.event_type, + "source": event.source, + "schema_version": event.schema_version, + "span_id": event.span_id, + "parent_span_id": str(event.parent_span_id) if event.parent_span_id else None, + "request_id": event.request_id, + "model": event.model, + "latency_ms": event.latency_ms, + "prompt_tokens": event.prompt_tokens, + "completion_tokens": event.completion_tokens, + "total_tokens": event.total_tokens, + "finish_reason": event.finish_reason, + "tool_name": event.tool_name, + "tool_arguments_length": event.tool_arguments_length, + "tool_result_length": event.tool_result_length, + "extra_json": event.extra_json, + "occurred_at": _iso(event.occurred_at), + "created_at": _iso(event.created_at), + } diff --git a/src/backend/routers/agent/memory.py b/src/backend/routers/agent/memory.py new file mode 100644 index 0000000..399e377 --- /dev/null +++ b/src/backend/routers/agent/memory.py @@ -0,0 +1,217 @@ +"""Persistent per-session agent memory. + +Endpoints (mounted under ``/api/agent``): + + GET /memory/{session_id} read the stored memory snapshot + PUT /memory/{session_id} replace the stored memory snapshot + DELETE /memory/{session_id} forget a session's memory + +This is the capability group-5 had and group-21 didn't: agent conversation state +survives an IDE (or agent process) restart, so a developer who closes their +editor mid-task can pick the conversation back up instead of starting over. + +Ownership is server-derived from the ACP scope on every call, so a runtime can +only read or write memory for sessions belonging to its own user and project — +the ``session_id`` in the path is a lookup key, not an authorization claim. +""" + +from __future__ import annotations + +import json +import logging +import uuid +from datetime import datetime +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field + +from App import App +from backend.acp_authorization import AcpSessionAuthorization +from backend.Responses import JsonResponseWithStatus +from backend.routers.agent.acp_auth import require_acp_scope +from database import crud +from privacy import collection + +router = APIRouter() + + +class AgentMemorySnapshot(BaseModel): + """The runtime's serialized memory window. + + Messages are stored opaquely: the shape is the runtime's business, and + schema-checking it here would couple the backend to the adapter's internal + context format. + """ + + messages: list[dict[str, Any]] = Field(default_factory=list) + + +def _scope_owns(scope: AcpSessionAuthorization, memory) -> bool: + return ( + memory.owner_user_id == str(scope.user_id) + and memory.owner_project_id == str(scope.project_id) + ) + + +def _serialize(memory) -> dict[str, Any]: + try: + payload = json.loads(memory.memory_json) or {} + except (ValueError, TypeError): + logging.warning( + f"[Agent/memory] unparseable snapshot for session {memory.session_id}" + ) + payload = {} + messages = payload.get("messages", []) + if not isinstance(messages, list): + messages = [] + return { + "session_id": memory.session_id, + "messages": messages, + "updated_at": _iso(memory.updated_at), + "created_at": _iso(memory.created_at), + } + + +def _iso(value: Any) -> Any: + return value.isoformat() if isinstance(value, datetime) else value + + +@router.get("/memory/{session_id}", summary="Read persisted agent session memory") +def get_agent_memory( + session_id: str, + app: App = Depends(App.get_instance), + scope: AcpSessionAuthorization = Depends(require_acp_scope), +) -> JsonResponseWithStatus: + db = app.get_db_session() + try: + memory = crud.get_agent_memory_by_session_id(db, session_id) + if memory is None: + # A fresh session legitimately has no memory yet — the runtime + # treats 404 as "start empty", so this is not an error path. + raise HTTPException(status_code=404, detail="Agent memory not found.") + if not _scope_owns(scope, memory): + raise HTTPException( + status_code=403, + detail="Agent memory is not authorized for this ACP session.", + ) + return JsonResponseWithStatus(status_code=200, content=_serialize(memory)) + except HTTPException: + raise + except Exception as error: + logging.error( + f"[Agent/memory] error reading {session_id}: {error}", exc_info=True + ) + raise HTTPException( + status_code=500, detail="Server failed to retrieve agent memory." + ) + finally: + db.close() + + +@router.put("/memory/{session_id}", summary="Replace persisted agent session memory") +def upsert_agent_memory( + session_id: str, + body: AgentMemorySnapshot, + app: App = Depends(App.get_instance), + scope: AcpSessionAuthorization = Depends(require_acp_scope), +) -> JsonResponseWithStatus: + """Replace the snapshot for one session. + + Note this is *not* gated by ``store_agent_content``. Memory is operational + state the agent needs to keep working across restarts — it's the + conversation itself, not telemetry about it, and gating it would silently + break the feature rather than protect anything. It is scoped to its owner, + replaced wholesale on every write, and deletable via DELETE. + + An account that opted out of data collection is the exception: new snapshots + are acknowledged but not stored, so the running turn is unaffected. A + snapshot saved before the opt-out stays readable until the account erases + its data; after an erase the runtime starts fresh. + """ + db = app.get_db_session() + try: + # The share lock, held until the upsert commits, keeps a save from + # slipping past a concurrent erase. + if not collection.lock_collection_allowed(db, uuid.UUID(str(scope.user_id))): + db.rollback() + return JsonResponseWithStatus( + status_code=200, + content={ + "session_id": session_id, + "messages": [], + "message_count": len(body.messages), + "stored": False, + }, + ) + existing = crud.get_agent_memory_by_session_id(db, session_id) + if existing is not None and not _scope_owns(scope, existing): + raise HTTPException( + status_code=403, + detail="Agent memory is not authorized for this ACP session.", + ) + memory = crud.upsert_agent_memory( + db, + session_id=session_id, + owner_user_id=str(scope.user_id), + owner_project_id=str(scope.project_id), + messages=body.messages, + ) + return JsonResponseWithStatus( + status_code=200, + content={ + **_serialize(memory), + "message_count": len(body.messages), + }, + ) + except HTTPException: + raise + except Exception as error: + db.rollback() + logging.error( + f"[Agent/memory] error saving {session_id}: {error}", exc_info=True + ) + raise HTTPException( + status_code=500, detail="Server failed to save agent memory." + ) + finally: + db.close() + + +@router.delete("/memory/{session_id}", summary="Forget a session's agent memory") +def delete_agent_memory( + session_id: str, + app: App = Depends(App.get_instance), + scope: AcpSessionAuthorization = Depends(require_acp_scope), +) -> JsonResponseWithStatus: + """Delete a session's memory. + + Gives the runtime (and through it the user) a way to actually discard + conversation state, rather than only overwriting it. + """ + db = app.get_db_session() + try: + existing = crud.get_agent_memory_by_session_id(db, session_id) + if existing is None: + raise HTTPException(status_code=404, detail="Agent memory not found.") + if not _scope_owns(scope, existing): + raise HTTPException( + status_code=403, + detail="Agent memory is not authorized for this ACP session.", + ) + crud.delete_agent_memory(db, session_id) + return JsonResponseWithStatus( + status_code=200, content={"deleted": True, "session_id": session_id} + ) + except HTTPException: + raise + except Exception as error: + db.rollback() + logging.error( + f"[Agent/memory] error deleting {session_id}: {error}", exc_info=True + ) + raise HTTPException( + status_code=500, detail="Server failed to delete agent memory." + ) + finally: + db.close() diff --git a/src/backend/routers/agent/profiles.py b/src/backend/routers/agent/profiles.py new file mode 100644 index 0000000..d8da166 --- /dev/null +++ b/src/backend/routers/agent/profiles.py @@ -0,0 +1,646 @@ +"""Researcher-owned agent profiles plus the plugin-facing registry. + +Endpoints (mounted under ``/api/agent``): + + GET /available-tools tool names a profile may select + GET /profiles list the caller's profiles (admin: all) + POST /profiles create a profile + PUT /profiles/{id} update a profile + DELETE /profiles/{id} archive a profile + GET /registry profiles in the shape the plugin consumes + +A profile is a private, editable template owned by exactly one researcher +(``agent_profile.owner_user_id``). It selects an administrator-managed +``provider_connection`` and a ``model`` from that connection's allowed list; the +release it pins is the approved artifact. The provider endpoint and secret live +on the connection, never on the profile. Names are unique per owner. + +There is no client-chosen profile assignment: task/run assignment is server-authoritative (see +``agents.registry``). +""" + +from __future__ import annotations + +import json +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from sqlalchemy.exc import IntegrityError + +from agents.tools import ( + KNOWN_AGENT_TOOLS, + tools_for_framework, + validate_command_timeout_seconds, + validate_commands_denylist, + validate_harness_options, +) +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, + require_admin, +) +from backend.routers.research.access import ( + is_owner, + require_owner, + require_researcher, +) +from database import crud +from database.db_schemas import AgentProfile +from research.study.agents import store as registry_store +from research.study.agents.distributions import ( + ProfileConfigurationError, + distribution_supported_platforms, + resolve_distribution_view, +) +from research.study.agents.enums import MANAGED_RUNTIME_FRAMEWORK + +router = APIRouter() + +SUPPORTED_FRAMEWORKS = ("code4me2-agent", "goose", "codex") +SUPPORTED_APPROVAL_POLICIES = ("auto", "per_step", "suggestion_only") +#: Cap on a researcher-authored system prompt, in characters as submitted (the +#: profile editor enforces the same limit). Long enough for study instructions, +#: short enough to stay a bounded share of the managed runtime's context window. +SYSTEM_PROMPT_MAX_LENGTH = 4000 + + +def _models_for(connection: Any) -> list[str]: + try: + parsed = json.loads(getattr(connection, "models_json", None) or "[]") + except (json.JSONDecodeError, TypeError): + return [] + if not isinstance(parsed, list): + return [] + return [str(item) for item in parsed] + + +class AgentProfilePayload(BaseModel): + """A profile template payload. + + ``extra="forbid"`` deliberately rejects the retired fields + (``base_url``/``api_key_ref``/``distribution_mode``/``agent_package``/ + ``agent_command``): researchers select an authorized ``connection_id`` and a + model, never an endpoint or a secret reference. The retired + ``commands_allowlist`` is refused the same way: a profile lists the commands + the agent may not run (``commands_denylist``). + """ + + model_config = ConfigDict(extra="forbid") + + name: str = Field(..., min_length=1) + model: str = Field(..., min_length=1) + framework_version: str = Field(default="code4me2-agent") + # Administrator-managed provider connection this profile uses. + connection_id: uuid.UUID + # Exact approved artifact pin (agent_release.release_id). + release_id: Optional[str] = Field(default=None) + tools_json: str = Field(default="[]") + approval_policy: str = Field(..., min_length=1) + max_steps: int = Field(..., ge=1) + is_active: bool = Field(default=True) + temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) + max_context_tokens: Optional[int] = Field(default=None, ge=1) + # Researcher-authored system prompt (managed runtime only; a BYOA release + # refuses it). Null or blank = no prompt; stored trimmed. On an update an + # omitted field keeps the stored prompt for the managed runtime and clears + # it for a BYOA runtime (see update_agent_profile). + system_prompt: Optional[str] = Field( + default=None, max_length=SYSTEM_PROMPT_MAX_LENGTH + ) + # Built-in runtime command and harness settings (decision D-01; managed + # runtime only, a BYOA release refuses them). Null = not set: no command is + # blocked (a config row may still block some), the runtime's default + # timeout and harness defaults. An empty denylist blocks nothing and an + # empty harness_options object carries no switch, so both are stored as + # null. Each field's shape is checked here; the cross-field rule (a + # verify_command may not run a blocked program) runs on the merged profile + # in the profile↔release contract. On an update an omitted field keeps the + # stored value for the managed runtime and clears it for a BYOA runtime, + # exactly like system_prompt. + commands_denylist: Optional[list[str]] = Field(default=None) + command_timeout_seconds: Optional[int] = Field(default=None) + harness_options: Optional[dict[str, Any]] = Field(default=None) + + @field_validator("commands_denylist", mode="before") + @classmethod + def check_commands_denylist(cls, value: Any) -> Optional[list[str]]: + if value is None: + return None + return validate_commands_denylist(value) or None + + @field_validator("command_timeout_seconds", mode="before") + @classmethod + def check_command_timeout_seconds(cls, value: Any) -> Optional[int]: + # "before" so a bool, a float or a numeric string is refused instead + # of being coerced to an int. + if value is None: + return None + return validate_command_timeout_seconds(value) + + @field_validator("harness_options", mode="before") + @classmethod + def check_harness_options(cls, value: Any) -> Optional[dict[str, Any]]: + if value is None: + return None + options = validate_harness_options(value) + return options or None + + @field_validator("system_prompt") + @classmethod + def normalize_system_prompt(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return None + if "\x00" in value: + # PostgreSQL text cannot store NUL; refuse it as a 422, not a 500. + raise ValueError("system_prompt must not contain NUL characters") + candidate = value.strip() + return candidate or None + + @field_validator("release_id") + @classmethod + def normalize_release_id(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return None + candidate = value.strip() + if not candidate: + return None + if candidate.lower() == "latest": + raise ValueError("release_id must be an immutable pin, not 'latest'") + return candidate + + @field_validator("name", "model") + @classmethod + def validate_required_text(cls, value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError("value must not be blank") + return normalized + + @field_validator("tools_json") + @classmethod + def validate_tools_json(cls, value: str) -> str: + try: + parsed = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError("tools_json must be a valid JSON array") from exc + if not isinstance(parsed, list) or not all( + isinstance(item, str) for item in parsed + ): + raise ValueError("tools_json must be a JSON array of strings") + return json.dumps(parsed) + + @field_validator("framework_version") + @classmethod + def validate_framework(cls, value: str) -> str: + normalized = value.strip().lower() + if normalized not in SUPPORTED_FRAMEWORKS: + raise ValueError( + f"framework_version must be one of {', '.join(SUPPORTED_FRAMEWORKS)}" + ) + return normalized + + @field_validator("approval_policy") + @classmethod + def validate_approval_policy(cls, value: str) -> str: + normalized = value.strip().lower() + if normalized not in SUPPORTED_APPROVAL_POLICIES: + raise ValueError( + "approval_policy must be one of " + + ", ".join(SUPPORTED_APPROVAL_POLICIES) + ) + return normalized + + @model_validator(mode="after") + def validate_tools_for_framework(self): + selected = set(json.loads(self.tools_json)) + unknown = selected - set(tools_for_framework(self.framework_version)) + if unknown: + raise ValueError( + f"tools_json contains tools unsupported by {self.framework_version}: " + + ", ".join(sorted(unknown)) + ) + return self + + +def _authorize_connection( + db: Any, payload: AgentProfilePayload, current_user: AuthenticatedUser +) -> Any: + """Resolve the selected admin-managed connection and model allowlist.""" + connection = crud.get_provider_connection(db, payload.connection_id) + if connection is None: + raise HTTPException( + status_code=422, + detail={ + "code": "CONNECTION_UNRESOLVED", + "field": "connection_id", + "message": "the selected provider connection does not exist", + }, + ) + if not current_user.is_admin and not crud.provider_connection_is_available( + db, connection.connection_id, current_user.user_id + ): + raise HTTPException( + status_code=403, + detail={ + "code": "CONNECTION_NOT_AVAILABLE", + "message": ( + "the provider connection is not active or available" + ), + }, + ) + if payload.model not in _models_for(connection): + raise HTTPException( + status_code=422, + detail={ + "code": "MODEL_NOT_ALLOWED", + "field": "model", + "message": ( + f"model {payload.model!r} is not allowed by connection " + f"{connection.label!r}" + ), + }, + ) + return connection + + +def _configuration_error_detail( + exc: crud.ProfileReleaseError | ProfileConfigurationError, +) -> dict[str, Any]: + """Typed 422 body for a profile↔release contract violation. + + ``field`` is included when the violation names one (for example + ``BYOA_FIELD_UNSUPPORTED`` on ``max_context_tokens``), so the editor can + attach the error to that input. + """ + detail: dict[str, Any] = {"code": exc.code, "message": str(exc)} + field = getattr(exc, "field", "") + if field: + detail["field"] = field + return detail + + +def _release_for(db: Any, profile: AgentProfile): + """Rehydrate the registry release a profile pins, if any.""" + release_id = getattr(profile, "release_id", None) + if not release_id: + return None + row = registry_store.get_release(db, release_id) + if row is None: + return None + return registry_store.row_to_release(row) + + +def _connection_summary(db: Any, profile: AgentProfile) -> Optional[dict[str, Any]]: + if getattr(profile, "connection_id", None) is None: + return None + connection = crud.get_provider_connection(db, profile.connection_id) + if connection is None: + return {"connection_id": str(profile.connection_id), "label": None} + return { + "connection_id": str(connection.connection_id), + "label": connection.label, + "models": _models_for(connection), + "is_active": bool(connection.is_active), + } + + +def _model_priced(db: Any, profile: AgentProfile) -> Optional[bool]: + from research.budget.errors import PriceMissing + from research.budget.pricing import get_model_price + from research.study.agents.enums import METERED_FRAMEWORKS + + if str(getattr(profile, "framework_version", "") or "").lower() not in METERED_FRAMEWORKS: + return None + try: + get_model_price(db, getattr(profile, "connection_id", None), profile.model) + except PriceMissing: + return False + return True + + +def _profile_to_dict(db: Any, profile: AgentProfile) -> dict[str, Any]: + release = _release_for(db, profile) + view = resolve_distribution_view(profile, release) + return { + "profile_id": str(profile.profile_id), + "owner_user_id": str(profile.owner_user_id), + "name": profile.name, + "model": profile.model, + "framework_version": profile.framework_version, + "tools_json": profile.tools_json, + "approval_policy": profile.approval_policy, + "max_steps": profile.max_steps, + "is_active": profile.is_active, + "temperature": profile.temperature, + "max_context_tokens": profile.max_context_tokens, + "system_prompt": getattr(profile, "system_prompt", None), + "commands_denylist": getattr(profile, "commands_denylist", None), + "command_timeout_seconds": getattr(profile, "command_timeout_seconds", None), + "harness_options": getattr(profile, "harness_options", None), + "configuration_digest": getattr(profile, "configuration_digest", ""), + "connection": _connection_summary(db, profile), + # Whether the frozen model has a budget price on its connection + # (metered runtimes only; ``None`` for Codex). A metered arm without a + # price refuses every call, so the study form warns on this flag. + "model_priced": _model_priced(db, profile), + "release_id": view.release_id, + "release_version": view.version, + "verified": view.verified, + "supported_platforms": distribution_supported_platforms(release), + "created_at": ( + profile.created_at.isoformat() + if isinstance(profile.created_at, datetime) + else profile.created_at + ), + } + + +def _profile_to_registry_dict(profile: AgentProfile) -> dict[str, Any]: + """Shape a profile for the plugin: non-secret identity only.""" + try: + tools = json.loads(profile.tools_json or "[]") + except json.JSONDecodeError: + tools = [] + + env = { + "SECURITY_CONFIRMATION_MODE": profile.approval_policy, + "MAX_ITERATIONS": str(profile.max_steps), + "CODE4ME_AGENT_PROFILE": profile.name, + } + if not tools: + env["DISABLE_TOOLS"] = "true" + + return { + "name": profile.name, + "model": profile.model, + "framework_version": profile.framework_version, + "tools": tools, + "approval_policy": profile.approval_policy, + "max_steps": profile.max_steps, + "max_context_tokens": profile.max_context_tokens, + "is_active": profile.is_active, + "temperature": profile.temperature, + "release_id": getattr(profile, "release_id", None), + "env": env, + } + + +@router.get("/available-tools", summary="List tool names a profile may select") +def list_available_tools( + framework_version: Optional[str] = Query( + default=None, + description="Restrict to the tools relevant for one runtime " + "(code4me2-agent | goose | codex). Omit for the full catalogue.", + ), + current_user: AuthenticatedUser = Depends(get_current_user), +): + require_researcher(current_user) + tools = ( + tools_for_framework(framework_version) + if framework_version + else KNOWN_AGENT_TOOLS + ) + return JsonResponseWithStatus( + status_code=200, + content={ + "tools": sorted(tools), + "framework_version": framework_version, + "frameworks": list(SUPPORTED_FRAMEWORKS), + }, + ) + + +@router.get("/profiles", summary="List agent profiles") +def list_agent_profiles( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_researcher(current_user) + db = app.get_db_session() + try: + owner_scope = None if current_user.is_admin else current_user.user_id + profiles = crud.list_agent_profiles(db, owner_scope) + return JsonResponseWithStatus( + status_code=200, + content={ + "profiles": [_profile_to_dict(db, profile) for profile in profiles] + }, + ) + finally: + db.close() + + +@router.post("/profiles", summary="Create an agent profile") +def create_agent_profile( + payload: AgentProfilePayload, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_researcher(current_user) + db = app.get_db_session() + try: + _authorize_connection(db, payload, current_user) + profile = crud.create_agent_profile( + db, + owner_user_id=current_user.user_id, + name=payload.name, + model=payload.model, + framework_version=payload.framework_version, + connection_id=payload.connection_id, + release_id=payload.release_id, + tools_json=payload.tools_json, + approval_policy=payload.approval_policy, + max_steps=payload.max_steps, + is_active=payload.is_active, + temperature=payload.temperature, + max_context_tokens=payload.max_context_tokens, + system_prompt=payload.system_prompt, + commands_denylist=payload.commands_denylist, + command_timeout_seconds=payload.command_timeout_seconds, + harness_options=payload.harness_options, + ) + return JsonResponseWithStatus( + status_code=201, content={"profile": _profile_to_dict(db, profile)} + ) + except crud.ProfileLockedError as exc: + db.rollback() + raise HTTPException( + status_code=409, + detail={"code": "PROFILE_LOCKED", "message": str(exc)}, + ) from exc + except (crud.ProfileReleaseError, ProfileConfigurationError) as exc: + db.rollback() + raise HTTPException( + status_code=422, + detail=_configuration_error_detail(exc), + ) from exc + except IntegrityError as exc: + db.rollback() + raise HTTPException( + status_code=409, + detail="You already have a profile with that name", + ) from exc + finally: + db.close() + + +@router.get("/profiles/{profile_id}", summary="Fetch one agent profile") +def get_agent_profile( + profile_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_researcher(current_user) + db = app.get_db_session() + try: + profile = crud.get_agent_profile_by_id(db, profile_id) + if profile is None: + raise HTTPException(status_code=404, detail="Agent profile not found") + require_owner(current_user, profile.owner_user_id, subject="profile") + return JsonResponseWithStatus( + status_code=200, content={"profile": _profile_to_dict(db, profile)} + ) + finally: + db.close() + + +@router.put("/profiles/{profile_id}", summary="Update an agent profile") +def update_agent_profile( + profile_id: uuid.UUID, + payload: AgentProfilePayload, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Update a profile template. + + Editing a template never rewrites an existing study snapshot; active-study + links are locked by the repository before any mutation. + """ + require_researcher(current_user) + db = app.get_db_session() + try: + existing = crud.get_agent_profile_by_id(db, profile_id) + if existing is None: + raise HTTPException(status_code=404, detail="Agent profile not found") + require_owner(current_user, existing.owner_user_id, subject="profile") + _authorize_connection(db, payload, current_user) + # The system prompt and the built-in runtime's command/harness settings + # share one rule: an omitted field keeps the stored value for the + # managed runtime and clears it for a BYOA runtime (which can never + # hold one); an explicit null always clears it. + byoa_runtime = payload.framework_version != MANAGED_RUNTIME_FRAMEWORK + + def replaces(field: str) -> bool: + return field in payload.model_fields_set or byoa_runtime + + profile = crud.update_agent_profile( + db, + profile_id=profile_id, + name=payload.name, + model=payload.model, + framework_version=payload.framework_version, + tools_json=payload.tools_json, + approval_policy=payload.approval_policy, + max_steps=payload.max_steps, + is_active=payload.is_active, + temperature=payload.temperature, + max_context_tokens=payload.max_context_tokens, + system_prompt=payload.system_prompt, + commands_denylist=payload.commands_denylist, + command_timeout_seconds=payload.command_timeout_seconds, + harness_options=payload.harness_options, + connection_id=payload.connection_id, + release_id=payload.release_id, + update_connection_id=True, + update_release_id=True, + # A PUT is a full replacement: a null override clears the stored + # value instead of keeping the previous one. + update_temperature=True, + update_max_context_tokens=True, + # Except the system prompt (and the command/harness settings) when + # the field is omitted for the managed runtime: a client that does + # not show it (an older website, or a release the catalogue does + # not list) must not silently wipe an arm's setting. A BYOA runtime + # can never hold one, so there an omitted field clears it (the + # website omits them when switching a profile to goose/codex). An + # explicit null always clears it. + update_system_prompt=replaces("system_prompt"), + update_commands_denylist=replaces("commands_denylist"), + update_command_timeout_seconds=replaces("command_timeout_seconds"), + update_harness_options=replaces("harness_options"), + ) + return JsonResponseWithStatus( + status_code=200, content={"profile": _profile_to_dict(db, profile)} + ) + except crud.ProfileLockedError as exc: + db.rollback() + raise HTTPException( + status_code=409, + detail={"code": "PROFILE_LOCKED", "message": str(exc)}, + ) from exc + except (crud.ProfileReleaseError, ProfileConfigurationError) as exc: + db.rollback() + raise HTTPException( + status_code=422, + detail=_configuration_error_detail(exc), + ) from exc + except IntegrityError as exc: + db.rollback() + raise HTTPException( + status_code=409, + detail="You already have a profile with that name", + ) from exc + finally: + db.close() + + +@router.delete("/profiles/{profile_id}", summary="Retire an agent profile") +def delete_agent_profile( + profile_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Archive a profile; frozen study snapshots are never erased.""" + require_researcher(current_user) + db = app.get_db_session() + try: + existing = crud.get_agent_profile_by_id(db, profile_id) + if existing is None: + raise HTTPException(status_code=404, detail="Agent profile not found") + require_owner(current_user, existing.owner_user_id, subject="profile") + crud.delete_agent_profile(db, profile_id) + return JsonResponseWithStatus( + status_code=200, + content={"retired": True, "profile_id": str(profile_id)}, + ) + except crud.ProfileLockedError as exc: + db.rollback() + raise HTTPException( + status_code=409, + detail={"code": "PROFILE_LOCKED", "message": str(exc)}, + ) from exc + finally: + db.close() + + +@router.get("/registry", summary="List profiles available to the plugin") +def list_agent_registry( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_researcher(current_user) + db = app.get_db_session() + try: + owner_scope = None if current_user.is_admin else current_user.user_id + profiles = crud.list_agent_profiles(db, owner_scope) + return JsonResponseWithStatus( + status_code=200, + content={"profiles": [_profile_to_registry_dict(p) for p in profiles]}, + ) + finally: + db.close() diff --git a/src/backend/routers/agents.py b/src/backend/routers/agents.py new file mode 100644 index 0000000..f07f4fe --- /dev/null +++ b/src/backend/routers/agents.py @@ -0,0 +1,712 @@ +"""Agent task lifecycle and the inference relay, for the plugin-facing paths. + +Endpoints (mounted under ``/api/agent``): + + POST /task mint a task, assigning the A/B arm server-side + POST /task/{id}/close finalize a task and aggregate its telemetry + POST /task/{id}/telemetry bulk-upload OTel spans for one finished session + POST /inference the authed relay third-party agents talk through + +All four authenticate via the plugin's ``session_token`` cookie, and every task +operation re-checks that the task belongs to the calling session — so one +developer's plugin can't read or write another's telemetry. +""" + +from __future__ import annotations + +import logging +import uuid +from datetime import datetime, timezone +from typing import Any, Optional + +from fastapi import APIRouter, Cookie, Depends, HTTPException, Response +from fastapi.responses import JSONResponse +from pydantic import BaseModel, Field + +from agents import inference, lifecycle, registry +from agents import provider as provider_module +from App import App +from backend.routers.agent.consent import resolve_store_agent_content +from backend.routers.research import access +from database import crud +from privacy import collection +from research.budget.meter import InferenceMeter +from research.study.agents.enums import METERED_FRAMEWORKS +from research.telemetry.adapters import LegacyFact, record_legacy_facts +from research.telemetry.enums import CoverageState +from research.telemetry.models import Correlations, Coverage, EventMetrics + +router = APIRouter(tags=["Agent"]) + + +def _require_funded_task(db, task): + """Re-check live enrollment/window/kill switch for a study-funded task. + + Non-research operational tasks (no ``study_id``) are unaffected and yield + ``None``. A research task must resolve an ACTIVE enrollment in its own + study, with the study window open and no operator kill switch engaged at + the study scope or at that enrollment's scope (``access.require_funded_access`` + scopes the switch to the live enrollment). Returns the live enrollment row. + """ + if getattr(task, "study_id", None) is None: + return None + try: + return access.require_funded_access( + db, + account_id=getattr(task, "owner_user_id", None), + study_id=task.study_id, + ) + except access.FundedAccessRefused as exc: + raise HTTPException( + status_code=403, + detail={"code": exc.code, "message": exc.message}, + ) from exc + + +def require_session( + app: App = Depends(App.get_instance), + session_token: str = Cookie(""), +) -> uuid.UUID: + """Validate the session_token cookie via Redis and return the session_id UUID. + + The cookie value *is* the session_id primary key in the DB session table. + Raises 401 if the cookie is missing, expired, or malformed. + """ + redis_manager = app.get_redis_manager() + session_info = redis_manager.get("session_token", session_token) + if session_info is None or not session_info.get("user_token"): + raise HTTPException(status_code=401, detail="Invalid or expired session token") + try: + return uuid.UUID(session_token) + except ValueError: + raise HTTPException(status_code=401, detail="Malformed session token") + + +def _require_own_task(db, task_id: uuid.UUID, session_id: uuid.UUID): + """Load a task, enforcing that it belongs to the calling session.""" + task = crud.get_agent_task(db, task_id) + if task is None: + raise HTTPException(status_code=404, detail="Agent task not found") + if task.session_id is None or task.session_id != session_id: + logging.warning( + f"[Agent] 403 — task {task_id} session mismatch " + f"(task.session={str(task.session_id)[:8]}… vs " + f"cookie={str(session_id)[:8]}…)" + ) + raise HTTPException( + status_code=403, detail="Task does not belong to this session" + ) + return task + + +# ── Task creation ──────────────────────────────────────────────────────────── + + +class TaskCreateRequest(BaseModel): + task_id: Optional[uuid.UUID] = None + task_description: Optional[str] = None + # NOTE: the client does not choose its profile. The arm is assigned + # server-side (see registry.resolve_assignment) so A/B buckets can't be + # self-selected. Any `profile` field sent by an older client is ignored. + + +@router.post("/task") +def create_agent_task( + body: TaskCreateRequest, + app: App = Depends(App.get_instance), + session_id: uuid.UUID = Depends(require_session), +) -> JSONResponse: + """Mint an agent task with the assigned profile snapshotted onto it. + + Accepting a client-supplied ``task_id`` makes this idempotent: the plugin + generates the id before launching the agent, and a retry after a flaky + response returns 200 with the same task rather than creating a duplicate. + A task id already claimed by a *different* session is a 409, not a silent + takeover. + """ + logging.info( + f"[Agent/task] request — task_id={body.task_id} " + f"session={str(session_id)[:8]}…" + ) + + db = app.get_db_session() + try: + if body.task_id is not None: + existing = crud.get_agent_task(db, body.task_id) + if existing is not None: + if existing.session_id != session_id: + logging.warning( + f"[Agent/task] 409 — task {body.task_id} belongs to another " + f"session" + ) + raise HTTPException( + status_code=409, + detail="Task ID already exists under a different session " + "— use a new task_id", + ) + logging.info( + f"[Agent/task] idempotent — task {body.task_id} already owned " + f"by this session" + ) + return JSONResponse( + { + "task_id": str(existing.task_id), + "framework_version": existing.framework_version, + "agent_profile": existing.agent_profile, + "model": existing.model, + "agent_launch": { + "approval_policy": existing.approval_policy, + }, + }, + status_code=200, + ) + + session = crud.get_session_by_id(db, session_id) + if session is None or session.user_id is None: + raise HTTPException( + status_code=401, detail="Session is not associated with a user" + ) + + assignment = registry.resolve_assignment_context(db, session.user_id) + if assignment is None: + raise HTTPException( + status_code=503, + detail="No active agent profiles are configured on the server", + ) + profile = assignment.profile + # Serialize with a concurrent erase: from here to the task insert nothing + # commits (the assignment lookup above may, on first use), so a task either + # commits before the erase (and is erased with the rest) or waits for it and + # then sees the opt-out. + collection.lock_account(db, session.user_id) + + # task_description is the user's own words — content, so honour the + # consent gate even at creation time. The research enrollment gate is + # applied on top of the legacy preference. + binding = access.resolve_research_binding( + db, account_id=session.user_id, study_id=assignment.study_id + ) + content_included = resolve_store_agent_content( + db, + session_id, + study_id=assignment.study_id, + enrollment_id=binding.enrollment_id if binding else None, + ) + + task = crud.create_agent_task( + db, + agent_profile=profile.name, + model=profile.model, + approval_policy=profile.approval_policy, + tools_json=profile.tools_json, + temperature=profile.temperature, + framework_version=profile.framework_version, + task_description=body.task_description if content_included else None, + session_id=session_id, + owner_user_id=session.user_id, + funding_owner_user_id=getattr(profile, "funding_owner_user_id", None), + task_id=body.task_id, + source="plugin", + study_id=assignment.study_id, + study_assignment_id=assignment.assignment_id, + profile_id=profile.profile_id, + consent_content_storage=content_included, + research_session_id=binding.research_session_id if binding else None, + enrollment_id=binding.enrollment_id if binding else None, + ) + logging.info( + f"[Agent/task] created task_id={task.task_id} profile={profile.name!r} " + f"runtime={profile.framework_version} model={profile.model} " + f"user={str(session.user_id)[:8]}…" + ) + return JSONResponse( + { + "task_id": str(task.task_id), + # The plugin needs the runtime name to know which agent to launch. + "framework_version": profile.framework_version, + "agent_profile": profile.name, + # This is only the runtime's displayed/requested model. The inference relay + # still enforces the immutable task snapshot on every model call. + "model": task.model, + "agent_launch": { + "approval_policy": task.approval_policy, + }, + }, + status_code=201, + ) + except HTTPException: + raise + except Exception as e: + logging.error(f"[Agent/task] DB error creating task — {e}", exc_info=True) + raise HTTPException(status_code=500, detail="Failed to create task") + finally: + db.close() + + +# ── Inference relay ────────────────────────────────────────────────────────── + + +class InferenceRequest(BaseModel): + task_id: uuid.UUID + request: dict[str, Any] + enrichment: Optional[dict[str, Any]] = None + + +@router.post("/inference") +async def run_agent_inference( + body: InferenceRequest, + app: App = Depends(App.get_instance), + session_id: uuid.UUID = Depends(require_session), +) -> Response: + """Authed relay for third-party agents (Goose, Codex). + + The agent talks to the plugin's local proxy, which forwards here with the + session cookie attached. Upstream target, model, temperature and tool policy + all come from the profile snapshotted onto the task — not from what the + agent asked for — so the assigned A/B arm actually takes effect. + """ + logging.info( + f"[Agent/inference] request — task_id={body.task_id} " + f"session={str(session_id)[:8]}… " + f"model={body.request.get('model', '?')} " + f"stream={body.request.get('stream', False)} " + f"messages={len(body.request.get('messages', []))}" + ) + + db = app.get_db_session() + try: + task = _require_own_task(db, body.task_id, session_id) + # Funded research use requires a live enrollment and an open study + # window, re-checked here on every call (a previously issued capability + # never substitutes for current server state). + enrollment = _require_funded_task(db, task) + # Resolved server-side from the stored preference, never from the + # request body — see backend.routers.agent.consent. The research + # enrollment gate is applied on top. + content_included = resolve_store_agent_content( + db, + session_id, + study_id=task.study_id, + enrollment_id=task.enrollment_id, + ) + profile = task.profile + if profile is None and task.profile_id is not None: + profile = crud.get_agent_profile_by_id(db, task.profile_id) + # The connection grant belongs to the study's funding owner, never the + # participant (whose own admin flag must not widen access). + funding_owner_user_id, owner_is_admin = provider_module.funding_owner_for_task( + db, task + ) + try: + connection = provider_module.resolve_task_connection( + db, + profile, + funding_owner_user_id, + owner_is_admin=owner_is_admin, + ) + except provider_module.ProviderReadinessError as exc: + raise HTTPException( + status_code=503, + detail={"code": exc.code, "message": exc.message}, + ) from exc + task_snapshot = { + "agent_profile": task.agent_profile, + "model": task.model, + "temperature": task.temperature, + "tools_json": task.tools_json, + "framework_version": task.framework_version + or (profile.framework_version if profile else None), + "research_session_id": getattr(task, "research_session_id", None), + } + finally: + db.close() + + # A study-funded task may only relay Chat Completions calls from a metered + # runtime (Goose, built-in agent), and those are always metered against the + # participant's budget. Anything else on a study task (a Responses-shaped + # body, a Codex arm) is refused: the study's key is never spent unmetered. + # Non-study developer tasks are unmetered as before. + meter = None + if enrollment is not None: + framework = str(task_snapshot["framework_version"] or "").strip().lower() + chat_body = "input" not in body.request and isinstance(body.request.get("messages"), list) + if framework not in METERED_FRAMEWORKS or not chat_body: + raise HTTPException( + status_code=403, + detail={ + "code": "RELAY_NOT_METERED", + "message": ( + "study-funded relay calls must be Chat Completions requests " + "from a Goose or built-in agent arm" + ), + }, + ) + meter = InferenceMeter( + app=app, + enrollment_id=enrollment.enrollment_id, + study_id=task.study_id, + connection_id=connection.connection_id, + model=task_snapshot["model"], + entry_point="agent_inference", + research_session_id=task_snapshot["research_session_id"], + ) + + logging.info( + f"[Agent/inference] task validated — profile={task_snapshot['agent_profile']} " + f"runtime={task_snapshot['framework_version']} " + f"content={content_included} metered={meter is not None}" + ) + return await inference.run_inference( + task_uuid=body.task_id, + session_uuid=session_id, + openai_body=body.request, + enrichment=body.enrichment, + agent_profile=task_snapshot["agent_profile"], + model=task_snapshot["model"], + connection=connection, + temperature=task_snapshot["temperature"], + framework_version=task_snapshot["framework_version"], + profile_tools_json=task_snapshot["tools_json"], + content_included=content_included, + meter=meter, + app=app, + ) + + +# ── Telemetry upload ───────────────────────────────────────────────────────── + +# Maps OTel span names produced by the plugin's trace collectors to the +# event_type vocabulary already used in agent_event rows. +_SPAN_EVENT_TYPE: dict[str, str] = { + "agent.llm.invoke": "model_call", + "agent.tool.execute": "tool_call", + "agent.retrieval": "observation", +} + + +def _span_fact( + span: "SpanPayload", + event_type: str, + trace_id: Optional[str] = None, + model_call_id: Optional[str] = None, +) -> "LegacyFact": + """Build the canonical fact for one OTel child span (structural only). + + ``trace_id`` scopes the whole upload; ``model_call_id`` links one model + invocation with the tool calls it caused (the llm span's own id for model + events, the enclosing llm span for tools where the reporter provides it). + Both are opaque id strings, never content. + """ + attrs = span.attributes or {} + payload: dict[str, Any] = {} + model = attrs.get("llm.model") or attrs.get("model") + if model: + payload["model"] = model + if event_type == "tool_call" and attrs.get("tool.name"): + payload["tool_name"] = attrs.get("tool.name") + prompt = _safe_int(attrs.get("llm.prompt_tokens")) + completion = _safe_int(attrs.get("llm.completion_tokens")) + total = _safe_int(attrs.get("llm.total_tokens")) + counts: dict[str, int] = {} + if prompt is not None: + counts["prompt_tokens"] = prompt + if completion is not None: + counts["completion_tokens"] = completion + if total is not None: + counts["total_tokens"] = total + return LegacyFact( + kind=event_type, + occurred_at=datetime.now(timezone.utc), + payload=payload, + metrics=EventMetrics( + usage_tokens=total, + usage_capability=Coverage( + state=( + CoverageState.AVAILABLE + if total is not None + else CoverageState.UNAVAILABLE + ), + capability="usage", + reason=( + None + if total is not None + else "upstream did not report usage" + ), + ), + latency_ms=span.duration_ms if span.duration_ms > 0 else None, + counts=counts, + ), + coverage=Coverage(state=CoverageState.AVAILABLE, capability="span"), + correlations=Correlations( + trace_id=trace_id, + span_id=span.span_id, + parent_span_id=span.parent_span_id, + model_call_id=model_call_id, + ), + source_event_id=span.span_id, + emitter_id="proxy", + ) + + +def _safe_int(value: Any) -> Optional[int]: + """Coerce an OTel attribute value to int, or None if it isn't numeric.""" + try: + return int(value) + except (TypeError, ValueError): + return None + + +class SpanPayload(BaseModel): + trace_id: str + span_id: str + parent_span_id: Optional[str] = None + name: str + kind: str = "INTERNAL" + start_time: str + end_time: str + duration_ms: int = 0 + attributes: dict[str, Any] = Field(default_factory=dict) + status: str = "OK" + + +class TelemetryUploadRequest(BaseModel): + trace_id: str + agent_runtime: str + experiment_tags: dict[str, str] = Field(default_factory=dict) + spans: list[SpanPayload] + # NOTE: no `content_included` field. Whether content is stored is resolved + # server-side from the user's preference; a client-supplied flag here would + # be exactly the client-trusting design merge decision 3 rejects. + + +@router.post("/task/{task_id}/telemetry", status_code=204) +def upload_agent_telemetry( + task_id: uuid.UUID, + body: TelemetryUploadRequest, + app: App = Depends(App.get_instance), + session_id: uuid.UUID = Depends(require_session), +) -> Response: + """Persist OTel spans for one completed agent session. + + Span → DB mapping:: + + agent.task root span → updates AgentTask aggregates + agent.llm.invoke child span → AgentEvent(event_type='model_call') + agent.tool.execute child span → AgentEvent(event_type='tool_call') + agent.retrieval child span → AgentEvent(event_type='observation') + + Known span attributes (token counts, model, tool name) map onto columns; + the rest of the attribute bag goes into ``extra_json``. + + This is a bulk path for runtimes whose calls the proxy didn't see + individually. Where both fire for the same session, the proxy's per-call + rows are the more detailed record; these spans mainly recover the + task-level totals. + """ + logging.info( + f"[Agent/telemetry] upload — task={task_id} trace={body.trace_id} " + f"runtime={body.agent_runtime} spans={len(body.spans)}" + ) + + db = app.get_db_session() + try: + _require_own_task(db, task_id, session_id) + + root = next((s for s in body.spans if s.parent_span_id is None), None) + child_spans = [s for s in body.spans if s.parent_span_id is not None] + + if root is not None: + llm_spans = [s for s in child_spans if s.name == "agent.llm.invoke"] + total_prompt = sum( + _safe_int(s.attributes.get("llm.prompt_tokens")) or 0 + for s in llm_spans + ) + total_completion = sum( + _safe_int(s.attributes.get("llm.completion_tokens")) or 0 + for s in llm_spans + ) + crud.update_agent_task_status( + db, + task_id=task_id, + status="done", + total_steps=len(child_spans), + input_tokens=total_prompt or None, + output_tokens=total_completion or None, + ) + logging.info( + f"[Agent/telemetry] task {task_id} updated — " + f"steps={len(child_spans)} tokens={total_prompt}/{total_completion}" + ) + + task_row = crud.get_agent_task(db, task_id) + span_names = {span.span_id: span.name for span in body.spans} + span_facts = [] + for span in child_spans: + event_type = _SPAN_EVENT_TYPE.get(span.name, "observation") + # A model invocation is its own call; a tool belongs to the llm + # call that caused it when the reporter parents it there, + # otherwise the link is left null rather than invented. + if event_type == "model_call": + model_call_id: Optional[str] = span.span_id + elif ( + event_type == "tool_call" + and span.parent_span_id is not None + and span_names.get(span.parent_span_id) == "agent.llm.invoke" + ): + model_call_id = span.parent_span_id + else: + model_call_id = None + span_facts.append( + _span_fact( + span, + event_type, + trace_id=body.trace_id, + model_call_id=model_call_id, + ) + ) + if task_row is not None: + result = record_legacy_facts(db, task=task_row, facts=span_facts) + if result is not None: + # Research-bound task: the canonical writer is the only + # authority; a refused write is surfaced truthfully rather than + # silently degraded to the legacy table (ISSUE-07). + if not result.written: + status_code = 503 if result.retryable else 409 + raise HTTPException( + status_code=status_code, + detail={ + "code": result.reason, + "message": result.message, + }, + ) + logging.info( + f"[Agent/telemetry] canonicalized {len(child_spans)} span(s) " + f"for task {task_id}" + ) + return Response(status_code=204) + + base_index = ( + crud.reserve_agent_event_indexes(db, task_id, len(child_spans)) + if child_spans + else 0 + ) + for offset, span in enumerate(child_spans): + event_type = _SPAN_EVENT_TYPE.get(span.name, "observation") + attrs = span.attributes or {} + # OTel span ids are hex strings; only persist a parent link when it + # parses as a UUID, since that's the column type. + parent_span_uuid: Optional[uuid.UUID] = None + if span.parent_span_id: + try: + parent_span_uuid = uuid.UUID(str(span.parent_span_id)) + except (ValueError, TypeError): + parent_span_uuid = None + + crud.append_agent_event( + db, + task_id=task_id, + event_index=base_index + offset, + event_type=event_type, + source="proxy", + source_event_id=span.span_id, + ignore_duplicate_source=True, + latency_ms=span.duration_ms if span.duration_ms > 0 else None, + span_id=span.span_id, + parent_span_id=parent_span_uuid, + model=attrs.get("llm.model") or attrs.get("model"), + prompt_tokens=_safe_int(attrs.get("llm.prompt_tokens")), + completion_tokens=_safe_int(attrs.get("llm.completion_tokens")), + total_tokens=_safe_int(attrs.get("llm.total_tokens")), + tool_name=( + attrs.get("tool.name") if event_type == "tool_call" else None + ), + extra_json=_span_extra_json(span, body), + commit=False, + ) + db.commit() + logging.info( + f"[Agent/telemetry] stored {len(child_spans)} events for task {task_id}" + ) + return Response(status_code=204) + + except HTTPException: + raise + except Exception as e: + db.rollback() + logging.error( + f"[Agent/telemetry] DB error for task {task_id} — {e}", exc_info=True + ) + raise HTTPException(status_code=500, detail="Failed to store telemetry") + finally: + db.close() + + +def _span_extra_json(span: SpanPayload, body: TelemetryUploadRequest) -> str: + """Serialise the non-columnar span metadata into the JSON overflow column. + + Group-21's version dropped these attributes on the floor. Keeping them in + ``extra_json`` is the point of the overflow column: an unrecognised span + attribute costs a nested JSON key rather than being lost. + """ + import json as _json + + known = { + "llm.model", + "model", + "llm.prompt_tokens", + "llm.completion_tokens", + "llm.total_tokens", + "tool.name", + } + return _json.dumps( + { + "otel": { + "trace_id": span.trace_id, + "span_name": span.name, + "kind": span.kind, + "status": span.status, + "start_time": span.start_time, + "end_time": span.end_time, + # Raw parent id, since the column can only hold UUID-shaped ones. + "parent_span_id": span.parent_span_id, + }, + "agent_runtime": body.agent_runtime, + "experiment_tags": body.experiment_tags or None, + "attributes": { + k: v for k, v in (span.attributes or {}).items() if k not in known + } + or None, + }, + default=str, + ) + + +# ── Task close ─────────────────────────────────────────────────────────────── + + +@router.post("/task/{task_id}/close", status_code=204) +def close_agent_task( + task_id: uuid.UUID, + app: App = Depends(App.get_instance), + session_id: uuid.UUID = Depends(require_session), +) -> Response: + """Mark a task done and aggregate its telemetry into task-level totals. + + Called by the plugin on startup, closing the *previous* session's task, + because the plugin can't reliably detect when an ACP agent process exits. + Idempotent — safe to call repeatedly. + """ + db = app.get_db_session() + try: + _require_own_task(db, task_id, session_id) + summary = lifecycle.finalize_agent_task(db, task_id) + logging.info(f"[Agent/close] task {task_id} closed — {summary}") + return Response(status_code=204) + except HTTPException: + raise + except Exception as e: + db.rollback() + logging.error(f"[Agent/close] DB error for task {task_id} — {e}", exc_info=True) + raise HTTPException(status_code=500, detail="Failed to close task") + finally: + db.close() diff --git a/src/backend/routers/analytics/__init__.py b/src/backend/routers/analytics/__init__.py index 810a5bd..b184914 100644 --- a/src/backend/routers/analytics/__init__.py +++ b/src/backend/routers/analytics/__init__.py @@ -13,12 +13,13 @@ """ from fastapi import APIRouter -from . import usage, models, calibration, studies, overview +from . import agents, usage, models, calibration, studies, overview router = APIRouter() # Include all analytics sub-routers router.include_router(usage.router, prefix="/usage", tags=["Usage Analytics"]) +router.include_router(agents.router, prefix="/agents", tags=["Agent Analytics"]) router.include_router(models.router, prefix="/models", tags=["Model Analytics"]) router.include_router(calibration.router, prefix="/calibration", tags=["Model Calibration"]) router.include_router(studies.router, prefix="/studies", tags=["A/B Testing"]) diff --git a/src/backend/routers/analytics/agents.py b/src/backend/routers/analytics/agents.py new file mode 100644 index 0000000..022eae7 --- /dev/null +++ b/src/backend/routers/analytics/agents.py @@ -0,0 +1,82 @@ +"""Privacy-scoped agent analytics over the canonical event authority (phase 06). + +Both routes delegate to one owner-scoped canonical query path +(:mod:`research.analysis.read_models.dashboard`), which reads ``research_event`` +joined to ``agent_task`` by the explicit phase-05 binding +(``research_event.agent_run_id = agent_task.external_run_id``). Ownership is +enforced before the joins; there is no ACP-session-id or time-window bridging, +and a legacy run with no canonical events renders explicit unavailable values. +""" + +from datetime import datetime, timezone +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException, Query + +from App import App +from backend.Responses import JsonResponseWithStatus +from research.analysis.read_models.dashboard import agent_overview, agent_run_detail + +from .auth_utils import AuthenticatedUser, get_current_user + +router = APIRouter() + + +@router.get("/overview") +def get_agent_overview( + time_window: str = Query("7d", description="7d, 30d, or 90d"), + framework: Optional[str] = Query(None), + model: Optional[str] = Query(None), + profile: Optional[str] = Query(None), + user_id: Optional[str] = Query(None, description="Admin-only user filter"), + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Canonical, owner-scoped agent telemetry without content payloads.""" + db = app.get_db_session() + try: + content = agent_overview( + db, + current_user, + time_window=time_window, + framework=framework, + model=model, + profile=profile, + user_id=user_id, + ) + return JsonResponseWithStatus(status_code=200, content=content) + except ValueError as error: + raise HTTPException(status_code=400, detail=str(error)) from error + except HTTPException: + raise + except Exception as error: + db.rollback() + raise HTTPException( + status_code=500, detail=f"Error retrieving agent analytics: {error}" + ) from error + finally: + db.close() + + +@router.get("/runs/{task_id}") +def get_agent_run_detail( + task_id: str, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Return one authorized run's canonical event timeline, never content.""" + db = app.get_db_session() + try: + content = agent_run_detail(db, current_user, task_id=task_id) + if content is None: + raise HTTPException(status_code=404, detail="Agent run not found") + return JsonResponseWithStatus(status_code=200, content=content) + except HTTPException: + raise + except Exception as error: + db.rollback() + raise HTTPException( + status_code=500, detail=f"Error retrieving agent run: {error}" + ) from error + finally: + db.close() diff --git a/src/backend/routers/analytics/auth_utils.py b/src/backend/routers/analytics/auth_utils.py index 3a0f800..e115b40 100644 --- a/src/backend/routers/analytics/auth_utils.py +++ b/src/backend/routers/analytics/auth_utils.py @@ -21,12 +21,22 @@ class AuthenticatedUser: """Container for authenticated user information.""" - - def __init__(self, user_id: uuid.UUID, is_admin: bool, email: str, name: str): + + def __init__( + self, + user_id: uuid.UUID, + is_admin: bool, + email: str, + name: str, + can_research: bool = False, + ): self.user_id = user_id self.is_admin = is_admin self.email = email self.name = name + # Administrator-enabled researcher account (P3). There is no per-study + # role table; this single flag is the researcher authority. + self.can_research = can_research def get_current_user( @@ -69,7 +79,8 @@ def get_current_user( user_id=user.user_id, is_admin=user.is_admin, email=user.email, - name=user.name + name=user.name, + can_research=bool(getattr(user, "can_research", False)), ) except Exception as e: @@ -144,9 +155,23 @@ def validate_time_range(start_time: Optional[str], end_time: Optional[str]) -> t start_time = start_dt.isoformat() # Validate format - datetime.fromisoformat(start_time.replace('Z', '+00:00')) - datetime.fromisoformat(end_time.replace('Z', '+00:00')) - + start_dt = datetime.fromisoformat(start_time.replace('Z', '+00:00')) + end_dt = datetime.fromisoformat(end_time.replace('Z', '+00:00')) + + # An inverted range would silently return empty results, misleading + # researchers into thinking no telemetry was collected. Reject it. + try: + inverted = start_dt > end_dt + except TypeError: + # Mixed naive/aware bounds: not comparable in Python; leave the + # range to the database instead of failing the request. + inverted = False + if inverted: + raise HTTPException( + status_code=422, + detail="Invalid time range: start_time must not be after end_time", + ) + return start_time, end_time except ValueError: diff --git a/src/backend/routers/analytics/calibration.py b/src/backend/routers/analytics/calibration.py index 4e5ebb6..104ef8b 100644 --- a/src/backend/routers/analytics/calibration.py +++ b/src/backend/routers/analytics/calibration.py @@ -144,6 +144,8 @@ def get_reliability_diagram( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error generating reliability diagram: {str(e)}") @@ -190,39 +192,51 @@ def get_brier_score( group_col = group_columns[group_by] - query = f""" - SELECT - {group_col} as group_name, - mn.model_id, - COUNT(*) as sample_size, - -- Brier score: average of (probability - outcome)^2 - AVG(POWER(hg.confidence - CASE WHEN hg.was_accepted THEN 1.0 ELSE 0.0 END, 2)) as brier_score, - -- Decompose Brier score - AVG(CASE WHEN hg.was_accepted THEN 1.0 ELSE 0.0 END) as base_rate, - AVG(hg.confidence) as avg_confidence, - -- Reliability component: variance of confidence - accuracy per bin (simplified) - AVG(POWER(hg.confidence - AVG(CASE WHEN hg.was_accepted THEN 1.0 ELSE 0.0 END) OVER (), 2)) as reliability_component, - -- Resolution component: variance of bin accuracies (simplified) - VARIANCE(CASE WHEN hg.was_accepted THEN 1.0 ELSE 0.0 END) as resolution_component - FROM had_generation hg - JOIN meta_query mq ON hg.meta_query_id = mq.meta_query_id - JOIN model_name mn ON hg.model_id = mn.model_id - JOIN "user" u ON mq.user_id = u.user_id - JOIN config c ON u.config_id = c.config_id - LEFT JOIN contextual_telemetry ct ON mq.contextual_telemetry_id = ct.contextual_telemetry_id - LEFT JOIN programming_language pl ON ct.language_id = pl.language_id - WHERE mq.timestamp BETWEEN :start_time AND :end_time - AND hg.confidence IS NOT NULL - """ - + filters = "" if query_params.get("user_id"): - query += " AND mq.user_id = :user_id" + filters += " AND mq.user_id = :user_id" if model_id: - query += " AND hg.model_id = :model_id" + filters += " AND hg.model_id = :model_id" query_params["model_id"] = model_id - - query += f""" - GROUP BY {group_col}, mn.model_id + + # PostgreSQL rejects a window function nested inside an aggregate + # (the previous AVG(... AVG(...) OVER () ...) form always failed with a + # 500), so the overall base rate is computed once in a CTE instead. + query = f""" + WITH scored AS ( + SELECT + {group_col} AS group_name, + mn.model_id AS model_id, + hg.confidence AS confidence, + CASE WHEN hg.was_accepted THEN 1.0 ELSE 0.0 END AS outcome + FROM had_generation hg + JOIN meta_query mq ON hg.meta_query_id = mq.meta_query_id + JOIN model_name mn ON hg.model_id = mn.model_id + JOIN "user" u ON mq.user_id = u.user_id + JOIN config c ON u.config_id = c.config_id + LEFT JOIN contextual_telemetry ct ON mq.contextual_telemetry_id = ct.contextual_telemetry_id + LEFT JOIN programming_language pl ON ct.language_id = pl.language_id + WHERE mq.timestamp BETWEEN :start_time AND :end_time + AND hg.confidence IS NOT NULL{filters} + ), + overall AS ( + SELECT AVG(outcome) AS base FROM scored + ) + SELECT + scored.group_name AS group_name, + scored.model_id AS model_id, + COUNT(*) AS sample_size, + -- Brier score: average of (probability - outcome)^2 + AVG(POWER(scored.confidence - scored.outcome, 2)) AS brier_score, + -- Decompose Brier score + AVG(scored.outcome) AS base_rate, + AVG(scored.confidence) AS avg_confidence, + -- Reliability component: spread of confidence around the overall base rate (simplified) + AVG(POWER(scored.confidence - overall.base, 2)) AS reliability_component, + -- Resolution component: variance of outcomes (simplified) + VARIANCE(scored.outcome) AS resolution_component + FROM scored CROSS JOIN overall + GROUP BY scored.group_name, scored.model_id ORDER BY brier_score ASC """ @@ -261,6 +275,8 @@ def get_brier_score( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error calculating Brier score: {str(e)}") @@ -385,6 +401,8 @@ def get_confidence_distribution( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error retrieving confidence distribution: {str(e)}") @@ -485,6 +503,8 @@ def get_calibration_summary( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error retrieving calibration summary: {str(e)}") diff --git a/src/backend/routers/analytics/models.py b/src/backend/routers/analytics/models.py index a1f1e86..d340032 100644 --- a/src/backend/routers/analytics/models.py +++ b/src/backend/routers/analytics/models.py @@ -124,6 +124,8 @@ def get_model_comparison( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error retrieving model comparison: {str(e)}") @@ -217,6 +219,8 @@ def get_ground_truth_analysis( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error retrieving ground truth analysis: {str(e)}") @@ -329,6 +333,8 @@ def get_performance_by_context( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error retrieving performance by context: {str(e)}") @@ -412,6 +418,8 @@ def get_model_usage_trends( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error retrieving model usage trends: {str(e)}") diff --git a/src/backend/routers/analytics/overview.py b/src/backend/routers/analytics/overview.py index 4b13d14..8f9551e 100644 --- a/src/backend/routers/analytics/overview.py +++ b/src/backend/routers/analytics/overview.py @@ -249,6 +249,8 @@ def calculate_change(current, previous): content=dashboard_data ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error retrieving dashboard overview: {str(e)}") @@ -362,6 +364,8 @@ def get_activity_timeline( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error retrieving activity timeline: {str(e)}") @@ -499,6 +503,8 @@ def get_user_engagement_summary( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error retrieving user engagement: {str(e)}") diff --git a/src/backend/routers/analytics/studies.py b/src/backend/routers/analytics/studies.py index 6449dda..6df5661 100644 --- a/src/backend/routers/analytics/studies.py +++ b/src/backend/routers/analytics/studies.py @@ -11,7 +11,7 @@ """ from typing import Optional, List, Dict, Any -from datetime import datetime +from datetime import datetime, timezone import uuid import random @@ -32,9 +32,16 @@ class CreateStudyRequest(BaseModel): name: str description: Optional[str] = None starts_at: str # ISO datetime - ends_at: Optional[str] = None # ISO datetime + ends_at: Optional[str] = None # ISO datetime config_ids: List[int] # List of configuration IDs to test default_config_id: int # Configuration to use when study ends + # Optional agent-profile arms. Empty → a completion-only study, behaving + # exactly as before this field existed (backwards compatible). + agent_profile_ids: List[str] = [] # agent_profile UUIDs acting as A/B arms + # Which arm agent uplift is measured against; must be one of + # agent_profile_ids when set. Analogous to default_config_id on the + # completion side. + baseline_agent_profile_id: Optional[str] = None class UpdateStudyRequest(BaseModel): @@ -44,6 +51,13 @@ class UpdateStudyRequest(BaseModel): is_active: Optional[bool] = None +def _parse_study_datetime(value: str) -> datetime: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + @router.post("/create") def create_study( study_request: CreateStudyRequest, @@ -63,10 +77,10 @@ def create_study( try: # Validate datetime formats try: - starts_at = datetime.fromisoformat(study_request.starts_at.replace('Z', '+00:00')) + starts_at = _parse_study_datetime(study_request.starts_at) ends_at = None if study_request.ends_at: - ends_at = datetime.fromisoformat(study_request.ends_at.replace('Z', '+00:00')) + ends_at = _parse_study_datetime(study_request.ends_at) if ends_at <= starts_at: raise HTTPException(status_code=400, detail="End time must be after start time") except ValueError: @@ -77,23 +91,48 @@ def create_study( config = crud.get_config_by_id(db_session, config_id) if not config: raise HTTPException(status_code=400, detail=f"Configuration {config_id} not found") - + + # Validate the optional agent-profile arms. Done before any writes so a + # bad arm list can't leave a half-configured study behind. + # + # The legacy ``study_agent_profile`` arm authority was removed by the + # wave-1 consolidation (profiles are researcher-owned templates, and + # agent allocation is enrollment-scoped via the research + # ``study_assignment`` authority). Completion studies keep working, but + # they can no longer attach agent-profile arms through this route. + try: + agent_profile_uuids = [ + uuid.UUID(pid) for pid in study_request.agent_profile_ids + ] + except (ValueError, TypeError): + raise HTTPException(status_code=400, detail="Invalid agent profile ID format") + if agent_profile_uuids or study_request.baseline_agent_profile_id is not None: + raise HTTPException( + status_code=400, + detail=( + "agent-profile arms are no longer supported on completion " + "studies; create a research study for agent conditions" + ), + ) + baseline_profile_uuid: Optional[uuid.UUID] = None + # Check if there's already an active study active_study_query = """ SELECT study_id FROM study WHERE is_active = true LIMIT 1 """ active_study = db_session.execute(text(active_study_query)).fetchone() - if active_study and starts_at <= datetime.now(): + now = datetime.now(timezone.utc) + if active_study and starts_at <= now: # Deactivate existing active study deactivate_query = """ - UPDATE study SET is_active = false WHERE is_active = true + UPDATE study SET is_active = false WHERE is_active = true AND is_research = false """ db_session.execute(text(deactivate_query)) # Create new study study_id = uuid.uuid4() - is_active = starts_at <= datetime.now() and (ends_at is None or ends_at > datetime.now()) + is_active = starts_at <= now and (ends_at is None or ends_at > now) create_query = """ INSERT INTO study (study_id, name, description, created_by, starts_at, ends_at, is_active, default_config_id, created_at) @@ -109,25 +148,34 @@ def create_study( "ends_at": ends_at, "is_active": is_active, "default_config_id": study_request.default_config_id, - "created_at": datetime.now() + "created_at": now }) # If study is active, assign configurations to users if is_active: assign_users_to_study(db_session, study_id, study_request.config_ids) - + db_session.commit() - + return JsonResponseWithStatus( status_code=201, content={ "study_id": str(study_id), "name": study_request.name, "is_active": is_active, + "agent_profile_ids": [str(p) for p in agent_profile_uuids], + "baseline_agent_profile_id": ( + str(baseline_profile_uuid) if baseline_profile_uuid else None + ), "message": "Study created successfully" } ) + except HTTPException: + db_session.rollback() + raise + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error creating study: {str(e)}") @@ -244,6 +292,8 @@ def list_studies( content={"studies": studies} ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error listing studies: {str(e)}") @@ -326,6 +376,10 @@ def get_study_details( "engagement_rate": float(assignment.engagement_rate) if assignment.engagement_rate else 0.0 }) + # Agent-profile arms were removed by the wave-1 consolidation; a + # completion study has none. + agent_profiles: list[dict] = [] + return JsonResponseWithStatus( status_code=200, content={ @@ -340,10 +394,13 @@ def get_study_details( "default_config_id": study.default_config_id, "created_at": study.created_at.isoformat() }, - "assignments": assignment_data + "assignments": assignment_data, + "agent_profiles": agent_profiles } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error retrieving study details: {str(e)}") @@ -388,13 +445,13 @@ def activate_study( # Deactivate any currently active study deactivate_query = """ - UPDATE study SET is_active = false WHERE is_active = true + UPDATE study SET is_active = false WHERE is_active = true AND is_research = false """ db_session.execute(text(deactivate_query)) # Activate this study activate_query = """ - UPDATE study SET is_active = true WHERE study_id = :study_id + UPDATE study SET is_active = true WHERE study_id = :study_id AND is_research = false """ db_session.execute(text(activate_query), {"study_id": study_uuid}) @@ -419,6 +476,8 @@ def activate_study( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error activating study: {str(e)}") @@ -448,7 +507,7 @@ def deactivate_study( # Get study details study_query = """ SELECT study_id, name, default_config_id FROM study - WHERE study_id = :study_id AND is_active = true + WHERE study_id = :study_id AND is_active = true AND is_research = false """ study = db_session.execute(text(study_query), {"study_id": study_uuid}).fetchone() if not study: @@ -485,6 +544,8 @@ def deactivate_study( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error deactivating study: {str(e)}") @@ -523,15 +584,23 @@ def evaluate_study( if not study: raise HTTPException(status_code=404, detail="Study not found") - # Get study results by configuration + # Get study results by configuration. + # + # ``acceptance_rate`` here is the *legacy completion* acceptance + # (``had_generation.was_accepted``), a real completion-product producer — + # it is NOT the research edit acceptance (which has no producer and is + # rendered unavailable on the research surfaces). Unresolved generations + # (NULL ``was_accepted``) are excluded from the manual ratio rather than + # counted as rejections (L07). evaluation_query = """ SELECT cah.assigned_config_id, COUNT(DISTINCT cah.user_id) as total_users, COUNT(DISTINCT mq.user_id) as active_users, COUNT(mq.meta_query_id) as total_queries, - AVG(CASE WHEN hg.was_accepted THEN 1.0 ELSE 0.0 END) as acceptance_rate, - COUNT(CASE WHEN hg.was_accepted THEN 1 END) as total_accepted, + AVG(CASE WHEN hg.was_accepted THEN 1.0 END) + FILTER (WHERE hg.was_accepted IS NOT NULL) as acceptance_rate, + COUNT(CASE WHEN hg.was_accepted IS TRUE THEN 1 END) as total_accepted, COUNT(hg.meta_query_id) as total_generations, AVG(hg.generation_time) as avg_generation_time, AVG(hg.confidence) as avg_confidence, @@ -569,7 +638,11 @@ def evaluate_study( "activation_rate": (row.active_users or 0) / max(row.total_users, 1), "total_queries": row.total_queries or 0, "total_generations": row.total_generations or 0, - "acceptance_rate": float(row.acceptance_rate) if row.acceptance_rate else 0.0, + "acceptance_rate": ( + float(row.acceptance_rate) + if row.acceptance_rate is not None + else None + ), "total_accepted": row.total_accepted or 0, "avg_generation_time": float(row.avg_generation_time) if row.avg_generation_time else 0.0, "avg_confidence": float(row.avg_confidence) if row.avg_confidence else 0.0, @@ -593,14 +666,23 @@ def evaluate_study( config_acceptance = config_data["metrics"]["acceptance_rate"] config_generation_time = config_data["metrics"]["avg_generation_time"] - # Calculate uplift percentages - acceptance_uplift = ((config_acceptance - baseline_acceptance) / max(baseline_acceptance, 0.001)) * 100 + # Calculate uplift percentages (None when the acceptance + # rate is unavailable because no generation was decided). + acceptance_uplift = ( + ((config_acceptance - baseline_acceptance) / max(baseline_acceptance, 0.001)) * 100 + if config_acceptance is not None and baseline_acceptance is not None + else None + ) generation_time_change = ((config_generation_time - baseline_generation_time) / max(baseline_generation_time, 1)) * 100 config_data["vs_baseline"] = { "acceptance_rate_uplift_pct": acceptance_uplift, "generation_time_change_pct": generation_time_change, - "is_better_acceptance": config_acceptance > baseline_acceptance, + "is_better_acceptance": ( + config_acceptance > baseline_acceptance + if config_acceptance is not None and baseline_acceptance is not None + else None + ), "is_faster": config_generation_time < baseline_generation_time } @@ -618,9 +700,233 @@ def evaluate_study( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error evaluating study: {str(e)}") finally: db_session.close() + + +@router.get("/{study_id}/agent-evaluation") +def evaluate_study_agents( + study_id: str, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance) +): + """ + Evaluate an A/B study's *agent* arms. + + The agent-side mirror of ``evaluate_study``: aggregates canonical agent + telemetry per snapshotted profile and reports uplift against the baseline + arm. + + A task counts only when its immutable ``study_id`` and ``profile_id`` + snapshots identify this study. Legacy tasks without those snapshots are + intentionally excluded rather than inferred from a mutable profile name or + a time window. Canonical events join their task by the explicit phase-05 + binding (``research_event.agent_run_id = agent_task.external_run_id``), never + by an ACP session id or a time window. + + Edit acceptance has no canonical producer, so it is reported unavailable + (see ``edit_acceptance_rate: None``) rather than inferred. + + Admin only endpoint. + """ + db_session = app.get_db_session() + + try: + study_uuid = uuid.UUID(study_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid study ID format") + + try: + study_query = """ + SELECT study_id, name, starts_at, ends_at + FROM study WHERE study_id = :study_id + """ + study = db_session.execute( + text(study_query), {"study_id": study_uuid} + ).fetchone() + if not study: + raise HTTPException(status_code=404, detail="Study not found") + + window = { + "study_id": study_uuid, + "starts_at": study.starts_at, + "ends_at": study.ends_at, + } + + # Per-profile task aggregates use the immutable execution snapshot. + # Legacy tasks without a study/profile snapshot are excluded. + task_query = """ + SELECT + t.profile_id, + t.agent_profile AS profile_name, + t.model, + t.framework_version, + false AS is_baseline, + COUNT(DISTINCT t.task_id) AS total_tasks, + COUNT(DISTINCT t.task_id) FILTER (WHERE t.status = 'done') AS completed_tasks, + COUNT(DISTINCT t.task_id) FILTER (WHERE t.status = 'failed') AS failed_tasks, + COUNT(DISTINCT COALESCE(t.owner_user_id, t.session_id)) AS total_participants, + AVG(t.total_steps) AS avg_steps, + SUM(t.input_tokens) AS total_input_tokens, + SUM(t.output_tokens) AS total_output_tokens + FROM agent_task t + WHERE t.study_id = :study_id AND t.profile_id IS NOT NULL + GROUP BY t.profile_id, t.agent_profile, t.model, t.framework_version + ORDER BY t.agent_profile, t.profile_id + """ + task_rows = db_session.execute(text(task_query), window).fetchall() + + # Canonical per-arm event aggregates. Events join their task by the + # explicit phase-05 binding (research_event.agent_run_id = + # agent_task.external_run_id), never by an ACP session id or time window. + event_query = """ + SELECT + t.profile_id, + COUNT(e.event_id) FILTER ( + WHERE e.envelope_json -> 'payload' ->> 'legacy_kind' = 'model_call' + ) AS model_calls, + COUNT(e.event_id) FILTER ( + WHERE e.envelope_json -> 'payload' ->> 'legacy_kind' = 'tool_call' + ) AS tool_calls, + COUNT(e.event_id) FILTER ( + WHERE e.envelope_json -> 'payload' ->> 'legacy_kind' = 'tool_failed' + ) AS failures, + AVG((e.envelope_json -> 'metrics' ->> 'latency_ms')::float) FILTER ( + WHERE e.envelope_json -> 'payload' ->> 'legacy_kind' = 'model_call' + ) AS avg_model_latency_ms, + SUM((e.envelope_json -> 'metrics' ->> 'usage_tokens')::bigint) FILTER ( + WHERE e.envelope_json -> 'payload' ->> 'legacy_kind' = 'model_call' + ) AS total_tokens + FROM agent_task t + LEFT JOIN research_event e ON e.agent_run_id = t.external_run_id + WHERE t.study_id = :study_id AND t.profile_id IS NOT NULL + GROUP BY t.profile_id + """ + event_rows = { + str(row.profile_id): row + for row in db_session.execute(text(event_query), window).fetchall() + } + + # Edit acceptance has no canonical producer (the contract records it only + # "when truly exposed"), so it is reported unavailable rather than + # computed from anything else. There is no second, legacy edit source. + edit_rows: dict[str, object] = {} + + def _f(value) -> float: + return float(value) if value is not None else 0.0 + + def _i(value) -> int: + return int(value) if value is not None else 0 + + arm_results = [] + baseline_arm = None + + for row in task_rows: + events = event_rows.get(str(row.profile_id)) + total_tasks = _i(row.total_tasks) + + arm = { + "profile_id": str(row.profile_id), + "profile_name": row.profile_name, + "model": row.model, + "framework_version": row.framework_version, + "is_baseline": bool(row.is_baseline), + "metrics": { + "total_tasks": total_tasks, + "completed_tasks": _i(row.completed_tasks), + "completion_rate": _i(row.completed_tasks) / max(total_tasks, 1), + "failed_tasks": _i(row.failed_tasks), + "total_participants": _i(row.total_participants), + "avg_steps": _f(row.avg_steps), + "total_input_tokens": _i(row.total_input_tokens), + "total_output_tokens": _i(row.total_output_tokens), + "model_calls": _i(events.model_calls) if events else 0, + "tool_calls": _i(events.tool_calls) if events else 0, + "failures": _i(events.failures) if events else 0, + "avg_model_latency_ms": ( + _f(events.avg_model_latency_ms) if events else 0.0 + ), + "total_tokens": _i(events.total_tokens) if events else 0, + # No canonical edit producer exists, so edit acceptance is + # explicitly unavailable (never computed from tool success). + "total_edits": None, + "accepted_edits": None, + "rejected_edits": None, + "modified_edits": None, + "edit_acceptance_rate": None, + }, + } + + if arm["is_baseline"]: + baseline_arm = arm + arm_results.append(arm) + + # Uplift vs the baseline arm, when one is designated. + if baseline_arm: + base = baseline_arm["metrics"] + for arm in arm_results: + if arm["is_baseline"]: + continue + metrics = arm["metrics"] + arm["uplift"] = { + "completion_rate_change_pct": _pct_change( + metrics["completion_rate"], base["completion_rate"] + ), + # Edit acceptance is unavailable (no canonical producer), so + # its uplift is reported as unknown rather than zero. + "edit_acceptance_change_pct": None, + "avg_steps_change_pct": _pct_change( + metrics["avg_steps"], base["avg_steps"] + ), + "latency_change_pct": _pct_change( + metrics["avg_model_latency_ms"], base["avg_model_latency_ms"] + ), + "is_better_completion": ( + metrics["completion_rate"] > base["completion_rate"] + ), + "is_better_acceptance": None, + "is_faster": ( + metrics["avg_model_latency_ms"] < base["avg_model_latency_ms"] + ), + } + + return JsonResponseWithStatus( + status_code=200, + content={ + "study": { + "study_id": str(study.study_id), + "name": study.name, + "starts_at": study.starts_at.isoformat(), + "ends_at": study.ends_at.isoformat() if study.ends_at else None, + }, + "results": arm_results, + }, + ) + + except HTTPException: + raise + except Exception as e: + db_session.rollback() + raise HTTPException( + status_code=500, detail=f"Error evaluating study agents: {str(e)}" + ) + finally: + db_session.close() + + +def _pct_change(value: float, baseline: float) -> float: + """Percentage change of ``value`` against ``baseline``. + + Returns 0.0 when the baseline is zero: with no baseline to compare against, + a made-up ratio would read as a real effect. The absolute metrics are + reported alongside, so a caller can see that the baseline arm had no data. + """ + if not baseline: + return 0.0 + return ((value - baseline) / baseline) * 100 diff --git a/src/backend/routers/analytics/usage.py b/src/backend/routers/analytics/usage.py index e9f36b3..2e3b034 100644 --- a/src/backend/routers/analytics/usage.py +++ b/src/backend/routers/analytics/usage.py @@ -8,6 +8,7 @@ - Session and user behavior patterns """ +import logging from typing import Optional, List, Dict, Any from datetime import datetime @@ -131,6 +132,8 @@ def get_queries_over_time( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error retrieving usage data: {str(e)}") @@ -241,6 +244,8 @@ def get_acceptance_rates( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error retrieving acceptance rates: {str(e)}") @@ -273,20 +278,7 @@ def get_latency_distribution( query_params["user_id"] = user_id query_params = apply_user_filter(query_params, current_user) - query = """ - SELECT - mn.model_name, - c.config_id, - hg.generation_time, - mq.total_serving_time, - PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY hg.generation_time) - OVER (PARTITION BY mn.model_name) as p50_generation, - PERCENTILE_CONT(0.90) WITHIN GROUP (ORDER BY hg.generation_time) - OVER (PARTITION BY mn.model_name) as p90_generation, - PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY hg.generation_time) - OVER (PARTITION BY mn.model_name) as p95_generation, - PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY hg.generation_time) - OVER (PARTITION BY mn.model_name) as p99_generation + base_from = """ FROM had_generation hg JOIN meta_query mq ON hg.meta_query_id = mq.meta_query_id JOIN model_name mn ON hg.model_id = mn.model_id @@ -294,43 +286,64 @@ def get_latency_distribution( JOIN config c ON u.config_id = c.config_id WHERE mq.timestamp BETWEEN :start_time AND :end_time """ - if query_params.get("user_id"): - query += " AND mq.user_id = :user_id" + base_from += " AND mq.user_id = :user_id" if model_id: - query += " AND hg.model_id = :model_id" + base_from += " AND hg.model_id = :model_id" query_params["model_id"] = model_id - - query += " ORDER BY hg.generation_time" - - result = db_session.execute(text(query), query_params).fetchall() - - # Group data by model + + # PERCENTILE_CONT is an ordered-set aggregate; PostgreSQL rejects it as + # a window function, so the percentiles come from one grouped query and + # the (bounded) raw sample from another. + percentile_query = """ + SELECT + mn.model_name, + PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY hg.generation_time) AS p50_generation, + PERCENTILE_CONT(0.90) WITHIN GROUP (ORDER BY hg.generation_time) AS p90_generation, + PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY hg.generation_time) AS p95_generation, + PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY hg.generation_time) AS p99_generation + """ + base_from + """ + GROUP BY mn.model_name + ORDER BY mn.model_name + """ + raw_query = """ + SELECT + mn.model_name, + c.config_id, + hg.generation_time, + mq.total_serving_time + """ + base_from + """ + ORDER BY hg.generation_time + LIMIT 1000 + """ + + percentile_rows = db_session.execute(text(percentile_query), query_params).fetchall() + raw_rows = db_session.execute(text(raw_query), query_params).fetchall() + model_stats = {} - raw_data = [] - - for row in result: - raw_data.append({ + for row in percentile_rows: + model_stats[row.model_name] = { + "model_name": row.model_name, + "p50": float(row.p50_generation) if row.p50_generation else 0, + "p90": float(row.p90_generation) if row.p90_generation else 0, + "p95": float(row.p95_generation) if row.p95_generation else 0, + "p99": float(row.p99_generation) if row.p99_generation else 0 + } + raw_data = [ + { "model_name": row.model_name, "config_id": row.config_id, "generation_time": row.generation_time, "total_serving_time": row.total_serving_time - }) - - if row.model_name not in model_stats: - model_stats[row.model_name] = { - "model_name": row.model_name, - "p50": float(row.p50_generation) if row.p50_generation else 0, - "p90": float(row.p90_generation) if row.p90_generation else 0, - "p95": float(row.p95_generation) if row.p95_generation else 0, - "p99": float(row.p99_generation) if row.p99_generation else 0 - } - + } + for row in raw_rows + ] + return JsonResponseWithStatus( status_code=200, content={ "percentiles": list(model_stats.values()), - "raw_data": raw_data[:1000], # Limit raw data for performance + "raw_data": raw_data, # Bounded by the query (LIMIT 1000) "filters": { "start_time": start_time, "end_time": end_time, @@ -340,9 +353,12 @@ def get_latency_distribution( } ) - except Exception as e: + except HTTPException: + raise + except Exception: db_session.rollback() - raise HTTPException(status_code=500, detail=f"Error retrieving latency data: {str(e)}") + logging.exception("Error retrieving latency distribution") + raise HTTPException(status_code=500, detail="Error retrieving latency distribution") finally: db_session.close() @@ -425,6 +441,8 @@ def get_user_behavior_metrics( } ) + except HTTPException: + raise except Exception as e: db_session.rollback() raise HTTPException(status_code=500, detail=f"Error retrieving behavior metrics: {str(e)}") diff --git a/src/backend/routers/chat/get.py b/src/backend/routers/chat/get.py index cb81599..d21f0b6 100644 --- a/src/backend/routers/chat/get.py +++ b/src/backend/routers/chat/get.py @@ -170,7 +170,7 @@ def __parse_chat_history(db_session, chat_history_data): model_name=model_name, completion=generation.completion, generation_time=generation.generation_time, - confidence=generation.confidence, + confidence=generation.confidence if generation.confidence is not None else 0.0, was_accepted=generation.was_accepted, ) ) diff --git a/src/backend/routers/chat/request.py b/src/backend/routers/chat/request.py index 69d636d..837b85b 100644 --- a/src/backend/routers/chat/request.py +++ b/src/backend/routers/chat/request.py @@ -20,6 +20,7 @@ JsonResponseWithStatus, ) from database import crud +from privacy import collection from response_models import ( ChatCompletionErrorItem, ChatCompletionItem, @@ -30,6 +31,11 @@ ) from utils import create_uuid, extract_secrets, redact_secrets + +def _wire_confidence(value): + """A number for the plugin's generated client; NULL is stored, not sent.""" + return float(value) if value is not None else 0.0 + router = APIRouter() @@ -66,7 +72,15 @@ def request_chat_completion( We don't want to be generating a completion based on the old state of the chat. """ overall_start = time.perf_counter() - logging.info(f"Chat completion request: {chat_completion_request.dict()}") + logging.info( + "Chat completion request: chat_id=%s models=%s messages=%d " + "prefix_chars=%d suffix_chars=%d", + chat_completion_request.chat_id, + chat_completion_request.model_ids, + len(chat_completion_request.messages), + len(chat_completion_request.context.prefix or ""), + len(chat_completion_request.context.suffix or ""), + ) db_auth = app.get_db_session() redis_manager = app.get_redis_manager() @@ -99,6 +113,10 @@ def request_chat_completion( status_code=401, content=InvalidOrExpiredProjectToken() ) + # An account that opted out of data collection still gets its answer, + # but nothing about the request is stored, whatever its store_* flags say. + collect = collection.is_collection_allowed(db_auth, uuid.UUID(str(user_id))) + t1 = time.perf_counter() logging.info(f"Auth check took {(t1 - t0) * 1000:.2f}ms") # Redact secrets from context @@ -189,7 +207,7 @@ def handle_model_completion(model_id, created_query_id_provided): # Invoke the chat model with messages # ensure that the chat_completion_model is of type ChatCompletionModel - if not isinstance(chat_completion_model, completion.ChatCompletionModel): + if not isinstance(chat_completion_model, completion.CLASSIC_CHAT_MODEL_TYPES): return ChatCompletionErrorItem( model_name=str(model.model_name), message="Model is not a ChatCompletionModel", @@ -221,17 +239,25 @@ def handle_model_completion(model_id, created_query_id_provided): messages_copy.append((role, enhanced_content)) else: messages_copy.append((role, content)) - logging.info(f"The messages considered by the chat:\n{messages_copy}") + logging.info(f"The chat model receives {len(messages_copy)} messages") # Temporarily replace the messages for this model's invocation original_messages = chat_completion_request.messages chat_completion_request.messages = messages_copy - completion_result = chat_completion_model.invoke( - chat_completion_request.to_langchain_messages() - ) - - # Restore the original messages - chat_completion_request.messages = original_messages + # A provider outage costs this model's item, not the whole request. + try: + completion_result = chat_completion_model.invoke( + chat_completion_request.to_langchain_messages() + ) + except Exception as error: + logging.warning( + f"Chat model {model_id} ({model.model_name}) failed: {error}", + exc_info=True, + ) + return ChatCompletionErrorItem(model_name=str(model.model_name)) + finally: + # Restore the original messages + chat_completion_request.messages = original_messages local_t3 = time.perf_counter() @@ -257,7 +283,7 @@ def handle_model_completion(model_id, created_query_id_provided): generation_time=completion_result["generation_time"], shown_at=[datetime.now().isoformat()], was_accepted=False, - confidence=completion_result.get("confidence", 0.0), + confidence=completion_result.get("confidence"), logprobs=completion_result.get("logprobs", []), ).dict(), created_query_id_provided, @@ -269,7 +295,9 @@ def handle_model_completion(model_id, created_query_id_provided): model_name=str(model.model_name), completion=completion_result["completion"], generation_time=completion_result["generation_time"], - confidence=completion_result.get("confidence", 0.0), + # The plugin client reads a number; the stored value stays + # NULL for provider models so analytics leave it out. + confidence=_wire_confidence(completion_result.get("confidence")), was_accepted=False, ) @@ -371,13 +399,19 @@ def handle_model_completion(model_id, created_query_id_provided): ) ) - # Chain tasks - chain( - group(*pre_query_tasks) if pre_query_tasks else None, - add_chat_task, - add_chat_query_task, - group(*add_generation_tasks), - ).apply_async(queue="db") + # Chain tasks. Any stage may be absent (all store_* flags off, or + # every model errored): chain only the present stages, since + # chain(None, ...) raises TypeError and would 500 a request whose + # telemetry/query rows are still worth persisting. + chain_steps = [] + if pre_query_tasks: + chain_steps.append(group(*pre_query_tasks)) + chain_steps.append(add_chat_task) + chain_steps.append(add_chat_query_task) + if add_generation_tasks: + chain_steps.append(group(*add_generation_tasks)) + if collect: + chain(*chain_steps).apply_async(queue="db") t5 = time.perf_counter() logging.info(f"Celery task prep and queuing took {(t5 - t4) * 1000:.2f}ms") diff --git a/src/backend/routers/completion/feedback.py b/src/backend/routers/completion/feedback.py index de5d0ff..86a2f48 100644 --- a/src/backend/routers/completion/feedback.py +++ b/src/backend/routers/completion/feedback.py @@ -1,4 +1,5 @@ import logging +import uuid from typing import Union from fastapi import APIRouter, Cookie, Depends @@ -17,6 +18,7 @@ NoAccessToProvideFeedbackError, ) from celery_app.tasks import db_tasks +from privacy import collection from response_models import ResponseFeedbackResponseData router = APIRouter() @@ -93,6 +95,20 @@ def submit_completion_feedback( status_code=401, content=InvalidOrExpiredProjectToken() ) + # An account that opted out of data collection stores no feedback (its + # queries were never stored either); the client still sees success. + if not collection.is_collection_allowed(db_session, uuid.UUID(str(user_id))): + return JsonResponseWithStatus( + status_code=200, + content=CompletionFeedbackPostResponse( + message="Feedback not stored: data collection is turned off.", + data=ResponseFeedbackResponseData( + meta_query_id=feedback.meta_query_id, + model_id=feedback.model_id, + ), + ), + ) + # Retrieve the meta query and verify ownership by the current user query = crud.get_meta_query_by_id(db_session, feedback.meta_query_id) if not query: diff --git a/src/backend/routers/completion/get.py b/src/backend/routers/completion/get.py index dd3b8c8..ec6f2b8 100644 --- a/src/backend/routers/completion/get.py +++ b/src/backend/routers/completion/get.py @@ -111,7 +111,11 @@ def get_completions_by_query( model_name=str(model.model_name) if model else "Unknown Model", completion=str(generation.completion), generation_time=int(str(generation.generation_time)), - confidence=float(str(generation.confidence)), + confidence=( + float(str(generation.confidence)) + if generation.confidence is not None + else 0.0 + ), ) ) diff --git a/src/backend/routers/completion/request.py b/src/backend/routers/completion/request.py index 7d19e3a..3d9371e 100644 --- a/src/backend/routers/completion/request.py +++ b/src/backend/routers/completion/request.py @@ -21,6 +21,7 @@ JsonResponseWithStatus, ) from database import crud +from privacy import collection from response_models import ( CompletionErrorItem, ResponseCompletionItem, @@ -28,6 +29,11 @@ ) from utils import create_uuid, extract_secrets, redact_secrets + +def _wire_confidence(value): + """A number for the plugin's generated client; NULL is stored, not sent.""" + return float(value) if value is not None else 0.0 + router = APIRouter() """ @@ -108,6 +114,10 @@ def request_completion( status_code=401, content=InvalidOrExpiredProjectToken() ) + # An account that opted out of data collection still gets its answer, + # but nothing about the request is stored, whatever its store_* flags say. + collect = collection.is_collection_allowed(db_auth, uuid.UUID(str(user_id))) + t1 = time.perf_counter() logging.info(f"Auth check took {(t1 - t0) * 1000:.2f}ms") @@ -207,16 +217,24 @@ def handle_model_completion(model_id): if completion_model is None: return CompletionErrorItem(model_name=str(model.model_name)) local_t2 = time.perf_counter() - # Invoke the model with redacted prefix and suffix - completion_result = completion_model.invoke( - { - "prefix": completion_request.context.prefix, - "suffix": completion_request.context.suffix, - "multi_file_context": multi_file_contexts, - "file_name": completion_request.context.file_name, - }, - stop_sequences=completion_request.stop_sequences, - ) + # Invoke the model with redacted prefix and suffix. A provider + # outage costs this model's item, not the whole request. + try: + completion_result = completion_model.invoke( + { + "prefix": completion_request.context.prefix, + "suffix": completion_request.context.suffix, + "multi_file_context": multi_file_contexts, + "file_name": completion_request.context.file_name, + }, + stop_sequences=completion_request.stop_sequences, + ) + except Exception as error: + logging.warning( + f"Completion model {model_id} ({model.model_name}) failed: {error}", + exc_info=True, + ) + return CompletionErrorItem(model_name=str(model.model_name)) local_t3 = time.perf_counter() logging.info( @@ -255,7 +273,9 @@ def handle_model_completion(model_id): model_name=str(model.model_name), completion=completion_result["completion"], generation_time=completion_result["generation_time"], - confidence=completion_result["confidence"], + # The plugin client reads a number; the stored value stays + # NULL for provider models so analytics leave it out. + confidence=_wire_confidence(completion_result["confidence"]), ) # Execute completion calls concurrently using thread pool @@ -310,12 +330,18 @@ def handle_model_completion(model_id): ) ) - # Chain Celery tasks: store context/telemetry -> add query -> add generations - chain( - group(*pre_query_tasks) if pre_query_tasks else None, - add_query_task, - group(*add_generation_tasks), - ).apply_async(queue="db") + # Chain Celery tasks: store context/telemetry -> add query -> add generations. + # Any stage may be absent (all store_* flags off, or every model + # errored): chain only the present stages, since chain(None, ...) + # raises TypeError and would 500 an otherwise successful request. + chain_steps = [] + if pre_query_tasks: + chain_steps.append(group(*pre_query_tasks)) + chain_steps.append(add_query_task) + if add_generation_tasks: + chain_steps.append(group(*add_generation_tasks)) + if collect: + chain(*chain_steps).apply_async(queue="db") t6 = time.perf_counter() logging.info(f"Celery task prep and queuing took {(t6 - t5) * 1000:.2f}ms") diff --git a/src/backend/routers/config/__init__.py b/src/backend/routers/config/__init__.py index f76f04b..6ea6c19 100644 --- a/src/backend/routers/config/__init__.py +++ b/src/backend/routers/config/__init__.py @@ -39,6 +39,9 @@ def _maybe_parse_json(s: str) -> Any: return s +# Both spellings are served directly: the website normalizes ``/api/config/`` to +# ``/api/config`` and a redirect between them fails CORS preflight cross-origin. +@router.get("", include_in_schema=False) @router.get("/", summary="List all configs") def list_configs( current_user: AuthenticatedUser = Depends(require_admin), @@ -56,6 +59,61 @@ def list_configs( db.close() +# Static paths are declared before the `/{config_id}` routes: routes match in +# declaration order, so `/languages` and `/models` would otherwise be parsed +# as a config id and rejected with 422. +@router.get("/languages", summary="List supported programming languages") +def list_languages( + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + langs = crud.get_all_programming_languages(db) + items = [{"language_id": int(l.language_id), "language_name": l.language_name} for l in langs] + # Also provide as a mapping name -> id for convenience + mapping = {l["language_name"]: l["language_id"] for l in items} + return JsonResponseWithStatus(status_code=200, content={"languages": items, "mapping": mapping}) + finally: + db.close() + + +@router.get("/models", summary="List available models in database") +def list_models( + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + models = crud.get_all_model_names(db) + items = [{"model_id": int(m.model_id), "model_name": m.model_name} for m in models] + return JsonResponseWithStatus(status_code=200, content={"models": items}) + finally: + db.close() + + +@router.get("/models/validate", summary="Validate a Hugging Face model name") +def validate_hf_model( + name: str = Query(..., description="Hugging Face repo id, e.g. org/model"), + current_user: AuthenticatedUser = Depends(require_admin), +): + if not name or "/" not in name: + # HF models are generally in org/model format; allow single segment too but warn + candidate = name.strip() + else: + candidate = name.strip() + + # Use httpx to check if the page exists + url = f"https://huggingface.co/{candidate}" + try: + resp = httpx.get(url, follow_redirects=True, timeout=5.0) + exists = resp.status_code == 200 + return JsonResponseWithStatus(status_code=200, content={"name": candidate, "exists": exists, "status_code": resp.status_code}) + except Exception as e: + # If network error, return a soft failure so UI can warn but not block + return JsonResponseWithStatus(status_code=200, content={"name": candidate, "exists": False, "error": str(e)}) + + @router.get("/{config_id}", summary="Get a config by id") def get_config( config_id: int, @@ -75,6 +133,7 @@ def get_config( db.close() +@router.post("", include_in_schema=False) @router.post("/", summary="Create a new config") def create_config( payload: ConfigCreate, @@ -126,59 +185,10 @@ def delete_config( if not ok: raise HTTPException(status_code=404, detail="Config not found") return JsonResponseWithStatus(status_code=200, content={"deleted": True, "config_id": int(config_id)}) + except HTTPException: + # A missing config is a 404, not a server error. + raise except Exception as e: return JsonResponseWithStatus(status_code=500, content={"error": str(e)}) finally: db.close() - - -@router.get("/languages", summary="List supported programming languages") -def list_languages( - current_user: AuthenticatedUser = Depends(require_admin), - app: App = Depends(App.get_instance), -): - db = app.get_db_session() - try: - langs = crud.get_all_programming_languages(db) - items = [{"language_id": int(l.language_id), "language_name": l.language_name} for l in langs] - # Also provide as a mapping name -> id for convenience - mapping = {l["language_name"]: l["language_id"] for l in items} - return JsonResponseWithStatus(status_code=200, content={"languages": items, "mapping": mapping}) - finally: - db.close() - - -@router.get("/models", summary="List available models in database") -def list_models( - current_user: AuthenticatedUser = Depends(require_admin), - app: App = Depends(App.get_instance), -): - db = app.get_db_session() - try: - models = crud.get_all_model_names(db) - items = [{"model_id": int(m.model_id), "model_name": m.model_name} for m in models] - return JsonResponseWithStatus(status_code=200, content={"models": items}) - finally: - db.close() - - -@router.get("/models/validate", summary="Validate a Hugging Face model name") -def validate_hf_model( - name: str = Query(..., description="Hugging Face repo id, e.g. org/model"), - current_user: AuthenticatedUser = Depends(require_admin), -): - if not name or "/" not in name: - # HF models are generally in org/model format; allow single segment too but warn - candidate = name.strip() - else: - candidate = name.strip() - - # Use httpx to check if the page exists - url = f"https://huggingface.co/{candidate}" - try: - resp = httpx.get(url, follow_redirects=True, timeout=5.0) - exists = resp.status_code == 200 - return JsonResponseWithStatus(status_code=200, content={"name": candidate, "exists": exists, "status_code": resp.status_code}) - except Exception as e: - # If network error, return a soft failure so UI can warn but not block - return JsonResponseWithStatus(status_code=200, content={"name": candidate, "exists": False, "error": str(e)}) diff --git a/src/backend/routers/project/activate.py b/src/backend/routers/project/activate.py index 214cc64..ef0f5d8 100644 --- a/src/backend/routers/project/activate.py +++ b/src/backend/routers/project/activate.py @@ -105,7 +105,8 @@ def activate_project( project_token = str(activate_project_request.project_id) # Attempt to get project info from Redis project_info = redis_manager.get("project_token", project_token) - logging.log(logging.INFO, f"Retrieved project info: {project_info}") + # Never log project_info itself: it carries the project's code. + logging.log(logging.INFO, f"Project {project_token} cached: {project_info is not None}") if not project_info: # Project not in Redis, fetch from database @@ -180,6 +181,7 @@ def activate_project( value=project_token, httponly=True, samesite="lax", + secure=app.get_config().cookie_secure, ) return response_obj except Exception as e: diff --git a/src/backend/routers/project/create.py b/src/backend/routers/project/create.py index af4c570..b42e966 100644 --- a/src/backend/routers/project/create.py +++ b/src/backend/routers/project/create.py @@ -143,6 +143,7 @@ def create_project( value=project_token, httponly=True, samesite="lax", + secure=app.get_config().cookie_secure, ) return response_obj except Exception as e: diff --git a/src/backend/routers/research/__init__.py b/src/backend/routers/research/__init__.py new file mode 100644 index 0000000..a258f79 --- /dev/null +++ b/src/backend/routers/research/__init__.py @@ -0,0 +1,131 @@ +"""Research-platform sub-router (Issue 01+). + +Mounted at ``/api/research``. The study protocol publication API is +researcher/admin-scoped: it defines experimental intent, so it is never exposed +to participants. Capability qualification is confined to the admin-only +``/packages`` operator/build surface; there is no public self-attestation route. +""" + +from fastapi import APIRouter + +from .agents import router as agents_router +from .bootstrap import router as bootstrap_router +from .budgets import router as budgets_router +from .inference import router as inference_router +from .join import router as join_router +from .operations import operations_router as pilot_operations_router +from .packages import packages_router +from .participants import router as participants_router +from .providers import router as providers_router +from .read_models import operations_router as read_operations_router +from .researchers import router as researchers_router +from .sessions import router as sessions_router +from .studies import router as studies_router +from .study_analytics import router as study_analytics_router +from .study_export import router as study_export_router +from .study_traces import router as study_traces_router +from .telemetry import router as telemetry_router + +router = APIRouter() +router.include_router( + studies_router, + prefix="/studies", + tags=["Research Studies"], +) +# Study-owner/administrator analytics read models: participants table, +# per-participant dashboard and arm comparison (``/{study_id}/analytics/...``). +router.include_router( + study_analytics_router, + prefix="/studies", + tags=["Research Study Analytics"], +) +# Study-owner/administrator chat traces (stored content, never cached): +# ``/{study_id}/enrollments/{enrollment_id}/trace?chat_id=...``. +router.include_router( + study_traces_router, + prefix="/studies", + tags=["Research Study Traces"], +) +# Study-owner/administrator raw data export (ZIP of CSV/JSONL + manifest): +# ``/{study_id}/export``. +router.include_router( + study_export_router, + prefix="/studies", + tags=["Research Study Export"], +) +# Participant budgets on the shared provider key (study owner/admin): +# study default, apply-default, per-enrollment balance/adjustments/ledger. +router.include_router( + budgets_router, + prefix="/studies", + tags=["Research Budgets"], +) +# Participant self-enrollment by shared join code. ``/join/{code}`` is readable +# by any authenticated user; ``POST /join`` enrolls the caller. +router.include_router( + join_router, + prefix="/join", + tags=["Research Onboarding"], +) +# Administrator enablement of researcher accounts (the single researcher +# authority; no per-study role grants). +router.include_router( + researchers_router, + prefix="/researchers", + tags=["Research Researchers"], +) +# Administrator-managed provider connections plus a researcher's granted subset. +router.include_router( + providers_router, + tags=["Research Provider Connections"], +) +router.include_router( + agents_router, + prefix="/agents", + tags=["Research Agents"], +) +router.include_router( + participants_router, + prefix="/participants", + tags=["Research Participants"], +) +router.include_router( + bootstrap_router, + prefix="/bootstrap", + tags=["Research Bootstrap"], +) +router.include_router( + sessions_router, + prefix="/sessions", + tags=["Research Sessions"], +) +router.include_router( + telemetry_router, + prefix="/telemetry", + tags=["Research Telemetry"], +) +# One operations/read API surface: the RBAC-scoped researcher read models +# (enrollment coverage, telemetry coverage and +# metrics) and the admin-only pilot/release/kill-switch operations are merged +# into a single router mounted exactly once under /operations. Their handler +# paths are unchanged; authorization remains per-route. +operations_router = APIRouter() +operations_router.include_router(read_operations_router) +operations_router.include_router(pilot_operations_router) +router.include_router( + operations_router, + prefix="/operations", + tags=["Research Operations"], +) +router.include_router( + packages_router, + prefix="/packages", + tags=["Research Packages"], +) +# The metered, server-keyed Chat Completions gateway a Goose study arm is +# pointed at (bearer = the inference capability minted at bootstrap). +router.include_router( + inference_router, + prefix="/inference", + tags=["Research Inference Gateway"], +) diff --git a/src/backend/routers/research/access.py b/src/backend/routers/research/access.py new file mode 100644 index 0000000..05afe3d --- /dev/null +++ b/src/backend/routers/research/access.py @@ -0,0 +1,312 @@ +"""Researcher/administrator authorization helpers (consolidation phase 01). + +There is exactly one researcher authority: the ``user.can_research`` flag that an +administrator enables (P3). There is no per-study role table. Study ownership is +``study.created_by == user.user_id``; profile ownership is +``agent_profile.owner_user_id``. Administrators bypass every ownership check. +No route may let a non-administrator change ``can_research`` or ``is_admin``. + +This is also the single funded-access gate (phase 03): every funded operation +(participant relay inference, managed ACP inference, self-report run +creation/replay, bootstrap session creation, ACP grant minting) resolves the same +current server state here — the account is a participant, has a live (``ACTIVE``) +enrollment, the research study window is open, and no operator kill switch is +engaged. Server time is authoritative; a previously issued capability never +substitutes for this live re-check. +""" + +from __future__ import annotations + +import logging +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Callable, Optional + +from fastapi import HTTPException + +from backend.routers.analytics.auth_utils import AuthenticatedUser +from database.db_schemas import Study as StudyRow +from database.db_schemas import ResearchStudyStatus +from database.research_schemas import ResearchSessionV1 as ResearchSessionRow +from research.participants import identity as identity_store +from research.participants.enums import EnrollmentStatus +from research.study.protocol import store as protocol_store + +__all__ = [ + "is_researcher", + "require_researcher", + "is_owner", + "require_owner", + "is_study_owner", + "require_study_owner", + "FundedAccessRefused", + "require_live_enrollment", + "ResearchBinding", + "resolve_research_binding", +] + + +def is_researcher(user: AuthenticatedUser) -> bool: + """Whether the caller may act as a researcher (admin or enabled account).""" + return bool( + getattr(user, "is_admin", False) or getattr(user, "can_research", False) + ) + + +def require_researcher(user: AuthenticatedUser) -> AuthenticatedUser: + """Reject a caller who is neither an administrator nor an enabled researcher.""" + if not is_researcher(user): + raise HTTPException( + status_code=403, + detail={ + "code": "RESEARCHER_REQUIRED", + "message": ( + "an administrator must enable this account for research" + ), + }, + ) + return user + + +def is_owner(user: AuthenticatedUser, owner_user_id) -> bool: + """Whether ``user`` owns a row (administrators own everything).""" + if getattr(user, "is_admin", False): + return True + if owner_user_id is None: + return False + return str(owner_user_id) == str(user.user_id) + + +def require_owner(user: AuthenticatedUser, owner_user_id, *, subject: str = "resource"): + """Reject a non-owner, non-administrator caller.""" + if not is_owner(user, owner_user_id): + raise HTTPException( + status_code=403, + detail={ + "code": "FORBIDDEN_OWNER", + "message": f"not authorized for this {subject}", + }, + ) + + +def is_study_owner(user: AuthenticatedUser, study_row) -> bool: + """Study ownership is ``study.created_by`` (admin bypasses).""" + return is_owner(user, getattr(study_row, "created_by", None)) + + +def require_study_owner(user: AuthenticatedUser, study_row): + """Reject a caller who neither owns the study nor is an administrator.""" + if not is_study_owner(user, study_row): + raise HTTPException( + status_code=403, + detail={ + "code": "FORBIDDEN_STUDY", + "message": "not authorized for this study", + }, + ) + + +# --------------------------------------------------------------------------- +# Funded-access gate (phase 03; folded from research/runtime/funding.py) +# --------------------------------------------------------------------------- + + +class FundedAccessRefused(Exception): + """A typed refusal for a funded path (mapped to HTTP 403 by routers).""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def require_live_enrollment( + db: Any, + *, + account_id: Optional[uuid.UUID], + study_id: Optional[uuid.UUID] = None, + now: Optional[datetime] = None, + kill_switch_check: Optional[Callable[[], bool]] = None, +): + """Return the account's live enrollment or raise :class:`FundedAccessRefused`. + + ``study_id`` (when given) must match the live enrollment's study, so a task + minted for one study cannot be funded by an enrollment in another. + """ + if account_id is None: + raise FundedAccessRefused( + "ACCOUNT_REQUIRED", "funded research use requires an authenticated account" + ) + + participant_row = identity_store.get_participant_by_account(db, account_id) + if participant_row is None: + raise FundedAccessRefused( + "NOT_A_PARTICIPANT", "this account has no research participant mapping" + ) + + active = None + for row in identity_store.list_enrollments(db, participant_row.participant_id): + if row.status == EnrollmentStatus.ACTIVE.value: + active = row + break + if active is None: + raise FundedAccessRefused( + "ENROLLMENT_NOT_ACTIVE", + "this account has no active study enrollment", + ) + + if study_id is not None and active.study_id != study_id: + raise FundedAccessRefused( + "ENROLLMENT_STUDY_MISMATCH", + "the active enrollment is not for this study", + ) + + study = db.get(StudyRow, active.study_id) + if getattr(study, "research_status", None) == ResearchStudyStatus.STUDY_STOPPED.value: + raise FundedAccessRefused( + "STUDY_STOPPED", + "the study has been stopped and cannot accept funded activity", + ) + if not protocol_store.research_study_is_open(study, _now(now)): + # Lazy terminal sweep: a study that ended or was terminated is marked + # completed here (COMPLETED enrollments, revoked sessions, bumped epoch) + # so a previously issued capability cannot keep funded access alive. + _sweep_ended_study(db, study, active.study_id, now=_now(now)) + raise FundedAccessRefused( + "STUDY_NOT_OPEN", + "the study is not currently open (not started, ended, or terminated)", + ) + + if kill_switch_check is not None and kill_switch_check(): + raise FundedAccessRefused( + "KILL_SWITCH_ENGAGED", + "an operator kill switch is engaged for this study", + ) + + return active + + +def require_funded_access( + db: Any, + *, + account_id: Optional[uuid.UUID], + study_id: Optional[uuid.UUID] = None, + now: Optional[datetime] = None, +): + """The single funded gate for every relay/grant path. + + Resolves the account's live enrollment (:func:`require_live_enrollment`) + and then evaluates the operator kill switch at both the study scope and the + enrollment scope of that very enrollment, so an enrollment-scoped switch is + never missed because a caller had no enrollment id at hand. Returns the + live enrollment row (``enrollment_id``/``study_id`` feed the budget meter). + """ + from research.analysis.operations import store as operations_store + + active = require_live_enrollment(db, account_id=account_id, study_id=study_id, now=now) + if operations_store.is_kill_switch_engaged( + db, study_id=active.study_id, enrollment_id=active.enrollment_id, now=now + ): + raise FundedAccessRefused( + "KILL_SWITCH_ENGAGED", + "an operator kill switch is engaged for this study or enrollment", + ) + return active + + +def _sweep_ended_study(db, study, study_id, *, now: datetime) -> None: + """Complete enrollments for a study that is no longer open for execution. + + Only sweeps when the study is an ended/terminated research study (not merely + pre-start), so a scheduled future study's pre-start enrollments are left + intact. + """ + if study is None or not getattr(study, "is_research", False): + return + if getattr(study, "research_status", None) != "ACTIVE": + return + ends_at = getattr(study, "ends_at", None) + ended = ends_at is not None and now > ends_at + terminated = not getattr(study, "is_active", False) + if not (ended or terminated): + return + try: + identity_store.complete_enrollments_for_study(db, study_id, now=now) + except Exception as error: # noqa: BLE001 - the refusal above is decisive + logging.warning( + "[Funding] could not complete enrollments for closed study %s: %s", + study_id, + error, + ) + + +# --------------------------------------------------------------------------- +# Explicit research attribution (phase 05) +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ResearchBinding: + """An agent task's explicit research attribution. + + Resolved server-side from the *authorized* account and the frozen + assignment's study; never supplied by a caller and never inferred from a + time window. ``research_session_id`` is ``None`` when the enrollment has no + single active session (none, or more than one open window) — an explicit + "unknown context" state rather than a guessed one. + """ + + enrollment_id: uuid.UUID + study_id: uuid.UUID + research_session_id: Optional[uuid.UUID] + + +def resolve_research_binding( + db: Any, *, account_id: Optional[uuid.UUID], study_id: Optional[uuid.UUID] +) -> Optional[ResearchBinding]: + """Resolve the account's live research attribution for ``study_id``. + + Returns ``None`` (no research context) when the account has no participant + mapping or no ACTIVE enrollment in that study. A single active session for + the enrollment is bound explicitly; zero or multiple active sessions leave + ``research_session_id`` unknown. + """ + if account_id is None or study_id is None: + return None + participant_row = identity_store.get_participant_by_account(db, account_id) + if participant_row is None: + return None + active = next( + ( + row + for row in identity_store.list_enrollments(db, participant_row.participant_id) + if row.status == EnrollmentStatus.ACTIVE.value + and getattr(row, "study_id", None) == study_id + ), + None, + ) + if active is None: + return None + + session_rows = ( + db.query(ResearchSessionRow) + .filter( + ResearchSessionRow.enrollment_id == active.enrollment_id, + ResearchSessionRow.state.notin_(("ended", "revoked")), + ) + .all() + ) + research_session_id = ( + session_rows[0].session_id if len(session_rows) == 1 else None + ) + return ResearchBinding( + enrollment_id=active.enrollment_id, + study_id=active.study_id, + research_session_id=research_session_id, + ) + diff --git a/src/backend/routers/research/agents.py b/src/backend/routers/research/agents.py new file mode 100644 index 0000000..0494e33 --- /dev/null +++ b/src/backend/routers/research/agents.py @@ -0,0 +1,691 @@ +"""Admin and researcher API for the agent registry and capability contract. + +Mounted under ``/api/research/agents``. Release import (manifest + verified +archive bytes), platform tests and operator views are admin-only; the +release catalogue and the distribution views are read-only, non-secret and +researcher-readable. The handlers are intentionally thin: identity derivation, +platform selection and coverage live in :mod:`research.study.agents.registry` +and :mod:`research.study.agents.manifest_import`, and the SQLAlchemy session is +supplied by ``App.get_db_session`` and passed to the store helpers. + +Release records are digest-pinned and immutable. Editing a release's bytes +requires a new ``release_id``. There is no unverified registration path: the +only way a release enters the catalogue is ``POST /releases/import`` with the +exact archive bytes, whose digests become the plugin's bootstrap pins. +""" + +from __future__ import annotations + +import json +import os +import shutil +import tempfile +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from pathlib import Path +from typing import Any, Optional +from urllib.parse import unquote, urljoin, urlsplit + +import httpx +from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile +from pydantic import BaseModel, Field +from sqlalchemy.exc import IntegrityError + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, + require_admin, +) +from backend.routers.research.access import require_researcher +from database import crud +from research.study.agents import store +from research.study.agents.distributions import ( + distribution_supported_platforms, + release_profile_configurability, + resolve_distribution_view, +) +from research.study.agents.enums import DistributionMode, RegistryReasonCode +from research.study.agents.manifest_import import ( + ManifestImportError, + build_manifest_release, +) +from research.study.agents.models import ( + AgentReleaseV1, # noqa: TC001 - FastAPI evaluates route annotations at runtime + CapabilitySnapshotV1, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) +from research.study.agents.registry import ( + AgentRegistry, + coverage_report, + verified_approval_options, +) + +router = APIRouter() + + +class ResolveArtifactRequest(BaseModel): + """Select the distribution artifact for one participant platform.""" + + os: str + arch: str + + +class SnapshotUploadRequest(BaseModel): + """Upload a declared-vs-observed capability snapshot for a release.""" + + snapshot: CapabilitySnapshotV1 + + +#: Chunk size used while streaming an uploaded archive to disk. +_IMPORT_CHUNK_BYTES = 1024 * 1024 + + +def _positive_int_env(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None or not str(raw).strip(): + return default + try: + value = int(raw) + except (TypeError, ValueError): + return default + return value if value > 0 else default + + +def import_limits() -> tuple[int, int]: + """``(max archive bytes, max total bytes)`` for one multipart import.""" + return ( + _positive_int_env("RESEARCH_IMPORT_MAX_ARCHIVE_BYTES", 256 * 1024 * 1024), + _positive_int_env("RESEARCH_IMPORT_MAX_TOTAL_BYTES", 1024 * 1024 * 1024), + ) + + +def _parse_manifest_field(raw: str) -> dict[str, Any]: + try: + parsed = json.loads(raw) + except (TypeError, ValueError) as error: + raise HTTPException( + status_code=422, + detail={ + "code": "INVALID_MANIFEST", + "message": f"the manifest field is not valid JSON: {error}", + "field": "manifest", + }, + ) from error + if not isinstance(parsed, dict): + raise HTTPException( + status_code=422, + detail={ + "code": "INVALID_MANIFEST", + "message": "the manifest field must be a JSON object", + "field": "manifest", + }, + ) + return parsed + + +def _registry_with_release(release: AgentReleaseV1) -> AgentRegistry: + registry = AgentRegistry() + registry.register_release(release) + return registry + + +def _issue_payload(issue: Any) -> dict[str, Any]: + return issue.model_dump(mode="json") if issue is not None else {} + + +def _release_for_profile(db: Any, profile: Any): + """Rehydrate a PACKAGED distribution's release, or ``None``.""" + release_id = getattr(profile, "release_id", None) + if not release_id: + return None + row = store.get_release(db, release_id) + if row is None: + return None + return store.row_to_release(row) + + +def _distribution_payload(db: Any, profile: Any) -> dict[str, Any]: + """The derived, read-only distribution view the UI consumes. + + Exposes the resolved ``release_id`` and ``version``, the DERIVED ``verified`` + boolean and the release's supported platforms. No secret is included: the + provider block carries only non-secret identity (the connection id), never + the endpoint URL or the secret reference/value. + """ + release = _release_for_profile(db, profile) + view = resolve_distribution_view( + profile, release, distribution_id=profile.profile_id + ) + release_row = ( + store.get_release(db, profile.release_id) + if getattr(profile, "release_id", None) + else None + ) + return { + "distribution_id": str(profile.profile_id), + "name": profile.name, + "distribution_mode": view.distribution_mode, + "release_id": view.release_id, + "release_version": view.version, + "verified": view.verified, + "supported_platforms": distribution_supported_platforms(release), + # Approval options the release's conformance evidence actually verifies; + # the editor must not offer an option this list does not contain. + "verified_approval_options": verified_approval_options( + getattr(release_row, "release_json", None) + if release_row is not None + else None + ), + "agent_package": view.agent_package, + "agent_command": view.agent_command, + "agent_command_args": view.agent_command_args, + "provider": { + "framework_version": profile.framework_version, + "model": profile.model, + "connection_id": ( + str(profile.connection_id) + if getattr(profile, "connection_id", None) is not None + else None + ), + }, + } + + +def _release_catalogue_entry(row: Any) -> dict[str, Any]: + """One registered release as a non-secret catalogue entry. + + Derived, read-only fields only: no artifact bytes, command line, endpoint or + credential material. ``qualification_status`` is recomputed from the stored + producer test results so legacy rows stay unqualified. + ``compatible_frameworks`` / ``configurable_fields`` / + ``required_bindings_missing`` tell the profile editor which runtimes may pin + the release and which profile fields actually govern it (field names only, + never binding keys, environment variable names or commands). + """ + release = store.row_to_release(row) + summary = store.release_summary(row) + return { + "release_id": release.release_id, + "version": release.version, + "agent_id": release.agent_id, + "distribution_mode": release.distribution_mode.value, + "qualification_status": release.qualification_status.value, + "supported_platforms": distribution_supported_platforms(release), + "verified_approval_options": verified_approval_options(row.release_json), + "tests": summary["tests"], + "is_byoa": release.is_byoa, + **release_profile_configurability(release, release_json=row.release_json), + } + + +@router.post( + "/releases/import", + summary="Import a release from a build manifest and its verified archives", +) +async def import_release( + manifest: str = Form(...), + archives: Optional[list[UploadFile]] = File(None), + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Import a build manifest **and its archive bytes** as a pinned release. + + Multipart: ``manifest`` is the producer manifest JSON and ``archives`` are + the exact ZIPs it declares (basenames). Every file is streamed to a bounded + temporary file, then its size and SHA-256 are recomputed and must match the + manifest. Exactly the declared basenames are required: a missing, duplicate + or unexpected upload, a digest/size mismatch, or a placeholder digest + rejects the entire import and **no release row is created**. + + The release identity is the canonical manifest digest; each artifact's + digest is the ZIP digest the plugin's bootstrap manifest pins. Admin-only; + this is the only way a release enters the catalogue. + """ + require_admin(current_user) + + parsed = _parse_manifest_field(manifest) + max_archive, max_total = import_limits() + uploaded = archives or [] + temp_root = Path(tempfile.mkdtemp(prefix="code4me-release-import-")) + stored: dict[str, Path] = {} + total_bytes = 0 + try: + for upload in uploaded: + name = upload.filename or "" + if not name or name in {".", ".."} or name != Path(name).name or "\\" in name: + raise HTTPException( + status_code=422, + detail={ + "code": "INVALID_UPLOAD", + "message": "every uploaded archive needs a filename", + "field": "archives", + }, + ) + if name in stored: + raise HTTPException( + status_code=422, + detail={ + "code": "DUPLICATE_ARCHIVE", + "message": f"archive {name!r} was uploaded more than once", + "field": "archives", + }, + ) + destination = temp_root / name + size = 0 + with destination.open("wb") as sink: + while True: + chunk = await upload.read(_IMPORT_CHUNK_BYTES) + if not chunk: + break + size += len(chunk) + total_bytes += len(chunk) + if size > max_archive: + raise HTTPException( + status_code=413, + detail={ + "code": "ARCHIVE_TOO_LARGE", + "message": ( + f"archive {name!r} exceeds the {max_archive}-byte " + "per-archive import limit" + ), + "field": "archives", + }, + ) + if total_bytes > max_total: + raise HTTPException( + status_code=413, + detail={ + "code": "IMPORT_TOO_LARGE", + "message": ( + f"the import exceeds the {max_total}-byte total limit" + ), + "field": "archives", + }, + ) + sink.write(chunk) + stored[name] = destination + + try: + plan = build_manifest_release(parsed, archives=stored) + except ManifestImportError as error: + raise HTTPException( + status_code=422, + detail={ + "code": error.code, + "message": error.message, + "field": error.field, + }, + ) from error + + db = app.get_db_session() + try: + rows = [] + created = False + for release in [plan.release, *plan.byoa_releases]: + existing = store.get_release(db, release.release_id) + if existing is not None and ( + existing.agent_id != release.agent_id + or existing.source_manifest_digest != release.source_manifest_digest + ): + raise HTTPException(status_code=409, detail={"code": "DUPLICATE_RELEASE"}) + result = AgentRegistry().register_release(release) + if not result.accepted: + raise HTTPException(status_code=422, detail=_issue_payload(result.issue)) + created |= existing is None + rows.append(store.upsert_release(db, release, commit=False)) + db.commit() + return JsonResponseWithStatus( + status_code=201 if created else 200, + content={ + "accepted": True, + "created": created, + "release": store.release_summary(rows[0]), + "releases": [store.release_summary(row) for row in rows], + "verified_artifacts": plan.verified_artifacts, + }, + ) + except IntegrityError as error: + db.rollback() + raise HTTPException(status_code=409, detail="release identity already exists") from error + except HTTPException: + db.rollback() + raise + finally: + db.close() + finally: + shutil.rmtree(temp_root, ignore_errors=True) + + +class ReleaseUrlImport(BaseModel): + manifest_url: str + archive_urls: list[str] = Field(min_length=1, max_length=16) + + +@router.post("/releases/import-url", summary="Import immutable release assets over HTTPS") +async def import_release_url( + payload: ReleaseUrlImport, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + allowed_hosts = { + host.strip().lower() for host in os.environ.get( + "RESEARCH_RELEASE_HOSTS", + "github.com,release-assets.githubusercontent.com,objects.githubusercontent.com", + ).split(",") if host.strip() + } + max_archive, max_total = import_limits() + total = 0 + uploads = [] + with tempfile.TemporaryDirectory(prefix="code4me-release-url-") as directory: + try: + async with httpx.AsyncClient(timeout=60, trust_env=False) as client: + for index, source in enumerate([payload.manifest_url, *payload.archive_urls]): + parsed = urlsplit(source) + name = unquote(parsed.path.rsplit("/", 1)[-1]) + if not name or name in {".", ".."} or "/" in name or "\\" in name: + raise HTTPException(422, detail={"code": "INVALID_UPLOAD"}) + if index and any(upload.filename == name for upload in uploads): + raise HTTPException(422, detail={"code": "DUPLICATE_ARCHIVE"}) + destination = Path(directory) / str(index) + for redirect in range(6): + parsed = urlsplit(source) + if (parsed.scheme != "https" or parsed.hostname not in allowed_hosts + or parsed.username or parsed.password or parsed.port not in (None, 443)): + raise HTTPException(422, detail={"code": "UNTRUSTED_RELEASE_URL"}) + async with client.stream("GET", source) as response: + if response.is_redirect: + if redirect == 5 or not response.headers.get("location"): + raise HTTPException(422, detail={"code": "INVALID_REDIRECT"}) + source = urljoin(source, response.headers["location"]) + continue + response.raise_for_status() + size = 0 + with destination.open("wb") as sink: + async for chunk in response.aiter_bytes(_IMPORT_CHUNK_BYTES): + size += len(chunk) + total += len(chunk) + if size > (max_archive if index else 1024 * 1024) or total > max_total: + raise HTTPException(413, detail={"code": "IMPORT_TOO_LARGE"}) + sink.write(chunk) + break + if index: + uploads.append(UploadFile(destination.open("rb"), filename=name)) + manifest = (Path(directory) / "0").read_text(encoding="utf-8") + return await import_release(manifest, uploads, current_user, app) + except (httpx.HTTPError, UnicodeError, ValueError) as error: + raise HTTPException(422, detail={"code": "RELEASE_DOWNLOAD_FAILED"}) from error + finally: + for upload in uploads: + await upload.close() + + +@router.get("/distributions", summary="List distributions and their derived release view") +def list_distributions( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """List every ``AgentProfile`` as a distribution with its resolved release. + + Readable by any authenticated user (it exposes no secret) so a researcher can + see which distributions are VERIFIED before authoring a condition. + """ + db = app.get_db_session() + try: + profiles = crud.list_agent_profiles(db) + return JsonResponseWithStatus( + status_code=200, + content={ + "distributions": [ + _distribution_payload(db, profile) for profile in profiles + ] + }, + ) + finally: + db.close() + + +@router.get( + "/distributions/{distribution_id}", + summary="Fetch one distribution's derived release view", +) +def get_distribution( + distribution_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + profile = crud.get_agent_profile_by_id(db, distribution_id) + if profile is None: + raise HTTPException(status_code=404, detail="Distribution not found") + return JsonResponseWithStatus( + status_code=200, content={"distribution": _distribution_payload(db, profile)} + ) + finally: + db.close() + + +@router.get( + "/release-catalogue", + summary="List releases for researcher profile authoring", +) +def list_release_catalogue( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """The profile-independent release catalogue, readable by any researcher. + + A fresh installation must be able to author its first ``AgentProfile`` + before any profile row exists, so this view is derived only from registered + releases (ISSUE-11). It is read-only and exposes no secret; importing and + qualifying releases remains administrator-only under ``/releases``. + """ + require_researcher(current_user) + db = app.get_db_session() + try: + rows = store.list_releases(db) + return JsonResponseWithStatus( + status_code=200, + content={"releases": [_release_catalogue_entry(row) for row in rows]}, + ) + finally: + db.close() + + +@router.get("/releases", summary="List registered agent releases") +def list_releases( + agent_id: Optional[str] = None, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + rows = store.list_releases(db, agent_id) + return JsonResponseWithStatus( + status_code=200, + content={"releases": [store.release_summary(row) for row in rows]}, + ) + finally: + db.close() + + +@router.get("/releases/{release_id}", summary="Fetch one registered release") +def get_release( + release_id: str, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + row = store.get_release(db, release_id) + if row is None: + raise HTTPException(status_code=404, detail="Agent release not found") + return JsonResponseWithStatus( + status_code=200, + content={ + "release": store.release_summary(row), + "model": store.row_to_release(row).model_dump(mode="json"), + }, + ) + finally: + db.close() + + +@router.post("/releases/{release_id}/disable", summary="Permanently disable a release") +def disable_release( + release_id: str, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + row = store.disable_release(db, release_id) + if row is None: + raise HTTPException(status_code=404, detail="Agent release not found") + return JsonResponseWithStatus(status_code=200, content={"release": store.release_summary(row)}) + finally: + db.close() + + +@router.post( + "/releases/{release_id}/resolve", + summary="Resolve the distribution artifact for a platform", +) +def resolve_artifact( + release_id: str, + payload: ResolveArtifactRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Return the exact platform artifact or a typed compatibility block.""" + require_admin(current_user) + + db = app.get_db_session() + try: + row = store.get_release(db, release_id) + if row is None: + raise HTTPException(status_code=404, detail="Agent release not found") + + release = store.row_to_release(row) + registry = _registry_with_release(release) + result = registry.resolve_distribution(release, payload.os, payload.arch) + if not result.resolved: + raise HTTPException( + status_code=409, detail=_issue_payload(result.issue) + ) + content: dict[str, Any] = { + "resolved": True, + "distribution_mode": result.distribution_mode.value, + "artifact": ( + result.artifact.model_dump(mode="json") + if result.artifact is not None + else None + ), + } + if result.distribution_mode == DistributionMode.BYOA_EXTERNAL: + # A BYOA release resolves to a participant-installable identity, not + # a digest-pinned artifact. + content["agent_command"] = result.agent_command + content["agent_command_args"] = result.agent_command_args + content["agent_package"] = result.agent_package + return JsonResponseWithStatus(status_code=200, content=content) + finally: + db.close() + + +@router.post( + "/releases/{release_id}/snapshots", + summary="Upload an immutable capability snapshot", +) +def upload_snapshot( + release_id: str, + payload: SnapshotUploadRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Store a declared-vs-observed snapshot linked to the exact release.""" + require_admin(current_user) + + if payload.snapshot.release_id != release_id: + raise HTTPException( + status_code=422, + detail="snapshot.release_id does not match the path release_id", + ) + + db = app.get_db_session() + try: + if store.get_release(db, release_id) is None: + raise HTTPException(status_code=404, detail="Agent release not found") + row = store.insert_snapshot(db, payload.snapshot) + return JsonResponseWithStatus( + status_code=201, + content={"snapshot": store.snapshot_summary(row)}, + ) + finally: + db.close() + + +@router.get( + "/releases/{release_id}/snapshots", + summary="List capability snapshots for a release", +) +def list_snapshots( + release_id: str, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + if store.get_release(db, release_id) is None: + raise HTTPException(status_code=404, detail="Agent release not found") + rows = store.list_snapshots(db, release_id) + return JsonResponseWithStatus( + status_code=200, + content={"snapshots": [store.snapshot_summary(row) for row in rows]}, + ) + finally: + db.close() + + +@router.get( + "/releases/{release_id}/coverage", + summary="Summarize declared vs observed capability coverage", +) +def coverage( + release_id: str, + snapshot_id: Optional[uuid.UUID] = None, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Return the coverage matrix for one snapshot (default: the latest).""" + require_admin(current_user) + + db = app.get_db_session() + try: + if store.get_release(db, release_id) is None: + raise HTTPException(status_code=404, detail="Agent release not found") + + rows = list(store.list_snapshots(db, release_id)) + if snapshot_id is not None: + rows = [ + row + for row in rows + if store.row_to_snapshot(row).snapshot_id == snapshot_id + ] + if not rows: + raise HTTPException( + status_code=404, detail="No capability snapshot found for release" + ) + + snapshot = store.row_to_snapshot(rows[-1]) + return JsonResponseWithStatus( + status_code=200, + content=coverage_report(snapshot).model_dump(mode="json"), + ) + finally: + db.close() diff --git a/src/backend/routers/research/bootstrap.py b/src/backend/routers/research/bootstrap.py new file mode 100644 index 0000000..9732e02 --- /dev/null +++ b/src/backend/routers/research/bootstrap.py @@ -0,0 +1,532 @@ +"""Participant bootstrap API: signed manifest. + +Mounted under ``/api/research/bootstrap``. Handlers are intentionally thin: the +assignment and bootstrap composition logic lives in +:mod:`research.runtime.assignment` and :mod:`research.runtime.bootstrap`. Participants +authenticate with the normal ``get_current_user`` flow and only ever obtain a +signed, secret-free manifest for their own active enrollment. +""" + +from __future__ import annotations + +import os +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import Any, Callable, Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, +) +from database import crud +from database.db_schemas import ResearchStudyStatus, Study as StudyRow +from research.analysis.operations import store as operations_store +from research.compatibility.enums import CapabilityId, CapabilityState +from research.compatibility.evaluate import evaluate_compatibility +from research.compatibility.models import ( + AcpCapabilityReceiptV1, # noqa: TC001 - FastAPI evaluates route annotations at runtime + AgentIdentity, + CompatibilityRequest, + EnvironmentTuple, + RequiredCapability, +) +from research.participants import identity as identity_store +from research.runtime.assignment import store as assignment_store +from research.runtime.assignment.enums import AllocationOutcome +from research.runtime.assignment.hashing import assignment_strategy, manual_override_enabled +from research.runtime.assignment.service import allocate +from research.runtime.bootstrap.models import ( + BootstrapAgentProfile, + BootstrapManifestV1, + ResearchSessionRef, +) +from research.runtime.bootstrap.capability import verify_capability +from research.runtime.bootstrap.signer import verify_manifest +from research.runtime.bootstrap.service import ( + BootstrapSigningContext, + compose_bootstrap, +) +from research.runtime.sessions import store as session_store +from research.runtime.sessions.enums import SessionState +from research.budget.settings import BudgetSettings +from research.runtime.sessions.models import ResearchSessionV1 +from research.study.agents import store as registry_store +from research.study.protocol import store as protocol_store +from research.runtime.assignment.models import StudyProfileSelection + +router = APIRouter() + +# The signing secret is mandatory: there is no development fallback. If it is +# unset the router refuses to issue or verify any capability/manifest rather +# than signing with a predictable, hardcoded secret. +# +# It is resolved at request time (never only at import): `main.py` loads the +# deployment `.env` before the routers, but a container that receives the +# variable late, or a test that patches it, must still be honoured. The two +# module attributes below are import-time snapshots kept as explicit overrides +# (tests patch them); production reads the live environment through +# :func:`signing_secret`. +BOOTSTRAP_SIGNING_SECRET: Optional[str] = os.environ.get("BOOTSTRAP_SIGNING_SECRET") or None +_SIGNER: Optional[BootstrapSigningContext] = None + + +def signing_secret() -> Optional[str]: + """The configured bootstrap signing secret, or ``None`` when unset. + + Resolution order: an explicitly installed signer (tests), the module-level + override, then the live environment. Blank values count as unset. + """ + if _SIGNER is not None and _SIGNER.secret and _SIGNER.secret.strip(): + return _SIGNER.secret + if BOOTSTRAP_SIGNING_SECRET and BOOTSTRAP_SIGNING_SECRET.strip(): + return BOOTSTRAP_SIGNING_SECRET + value = os.environ.get("BOOTSTRAP_SIGNING_SECRET", "") + return value if value.strip() else None + + +def _require_signer() -> BootstrapSigningContext: + """Return the configured signer or raise a typed refusal.""" + if _SIGNER is not None: + return _SIGNER + secret = signing_secret() + if secret is None: + raise HTTPException( + status_code=503, + detail={ + "code": "SIGNING_SECRET_MISSING", + "message": ( + "BOOTSTRAP_SIGNING_SECRET is not configured; refusing to " + "issue a bootstrap manifest" + ), + }, + ) + return BootstrapSigningContext( + secret=secret, + inference_capability_ttl_seconds=BudgetSettings.from_env().capability_ttl_seconds, + ) + + +class ManifestVerificationRequest(BaseModel): + manifest: BootstrapManifestV1 + enrollment_id: uuid.UUID + context_id: str = Field(min_length=1, max_length=200) + + +@router.post("/verify", summary="Authenticate a manifest for its owner and execution context") +def verify_bootstrap_manifest( + payload: ManifestVerificationRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Online HMAC verification; the signing secret never leaves the server. + + Consumers trust this authenticated HTTPS origin and must bind its response + to the exact manifest digest, enrollment and context they requested. + """ + signer = _require_signer() + manifest = payload.manifest + verification = verify_manifest(manifest, signer.secret) + if not verification.ok: + raise HTTPException(status_code=409, detail={"code": verification.reason.value}) + if manifest.enrollment_id != payload.enrollment_id: + raise HTTPException(status_code=403, detail="Manifest enrollment mismatch") + db = app.get_db_session() + try: + enrollment = _owned_enrollment(db, current_user, payload.enrollment_id) + session = session_store.get_session(db, manifest.research_session.research_session_id) + if ( + session is None + or session.enrollment_id != enrollment.enrollment_id + or session.study_id != enrollment.study_id + or manifest.study_id != enrollment.study_id + or session.context_id != payload.context_id + or SessionState(session.state).is_terminal + ): + raise HTTPException(status_code=409, detail="Manifest session is not active in this context") + capability = verify_capability( + manifest.session_capability, + signer.secret, + expected_audience="research-runtime", + expected_scope=["telemetry:write", "session:heartbeat", "session:close"], + now=_now(), + current_revocation_epoch=enrollment.revocation_epoch, + expected_enrollment_id=enrollment.enrollment_id, + expected_research_session_id=session.session_id, + expected_study_id=enrollment.study_id, + ) + if not capability.ok: + raise HTTPException(status_code=409, detail={"code": capability.reason.value}) + if _kill_switch_for_scope(db, study_id=enrollment.study_id, enrollment_id=enrollment.enrollment_id)(): + raise HTTPException(status_code=409, detail={"code": "KILL_SWITCH_ENGAGED"}) + return { + "verified": True, + "manifest_digest": manifest.manifest_digest, + "enrollment_id": str(enrollment.enrollment_id), + "context_id": payload.context_id, + } + finally: + db.close() + + +def _kill_switch_for_scope( + db: Any, + *, + study_id: Optional[uuid.UUID], + enrollment_id: Optional[uuid.UUID], +) -> Callable[[], bool]: + """DB-backed kill-switch predicate for one bootstrap scope.""" + return operations_store.db_kill_switch_check( + db, + study_id=study_id, + enrollment_id=enrollment_id, + ) + + +class EnvironmentReport(BaseModel): + """The participant's installed host tuple.""" + + os: str + arch: str + ide_build: Optional[str] = None + plugin_version: Optional[str] = None + host_kind: Optional[str] = None + # The JetBrains AI Assistant build the plugin found (the ACP host it + # depends on); reported so a zero-data participant can be diagnosed. + ai_assistant_version: Optional[str] = None + + +class ResearchSessionRequest(BaseModel): + """Request a bootstrap manifest for an active enrollment.""" + + enrollment_id: uuid.UUID + # Opaque execution-context id for this project/window. Required so two + # simultaneous windows get distinct sessions; identical values are + # idempotent. Never a filesystem path. + context_id: str = Field(min_length=1, max_length=200) + environment: EnvironmentReport + capability_receipt: Optional[AcpCapabilityReceiptV1] = None + compatibility_receipt_id: Optional[uuid.UUID] = None + prepare_only: bool = False + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _issue_payload(issue: Any) -> dict[str, Any]: + return issue.model_dump(mode="json") if issue is not None else {} + + +def _owned_enrollment(db: Any, current_user: AuthenticatedUser, enrollment_id: uuid.UUID): + participant_row = identity_store.get_participant_by_account(db, current_user.user_id) + if participant_row is None: + raise HTTPException(status_code=404, detail="Enrollment not found") + row = identity_store.get_enrollment(db, enrollment_id) + if row is None: + raise HTTPException(status_code=404, detail="Enrollment not found") + enrollment = identity_store.row_to_enrollment(row) + if enrollment.participant_id != participant_row.participant_id: + raise HTTPException(status_code=404, detail="Enrollment not found") + study = db.get(StudyRow, enrollment.study_id) + if getattr(study, "research_status", None) == ResearchStudyStatus.STUDY_STOPPED.value: + raise HTTPException( + status_code=403, + detail={ + "code": "STUDY_STOPPED", + "message": "the study has been stopped and cannot be bootstrapped", + }, + ) + if getattr(enrollment.status, "value", enrollment.status) != "ACTIVE": + raise HTTPException( + status_code=403, + detail={ + "code": "ENROLLMENT_NOT_ACTIVE", + "message": "the enrollment is not active", + }, + ) + return enrollment + + +def _evaluate_compatibility( + payload: ResearchSessionRequest, release: Any, db: Any +) -> tuple[Any, Optional[str]]: + """Evaluate compatibility from an inline receipt or a stored receipt id.""" + receipt = payload.capability_receipt + receipt_ref: Optional[str] = None + if receipt is not None: + receipt_ref = str(receipt.receipt_id) + elif payload.compatibility_receipt_id is not None: + row = _compatibility_store().get_receipt(db, payload.compatibility_receipt_id) + if row is not None: + receipt = _compatibility_store().row_to_receipt(row) + receipt_ref = str(payload.compatibility_receipt_id) + + if receipt is None: + return None, receipt_ref + + requirements: list[RequiredCapability] = [] + + expected_agent = None + if release is not None: + expected_agent = AgentIdentity( + agent_id=release.agent_id, + agent_version=release.version, + adapter_version=release.adapter.version if release.adapter else None, + ) + + request = CompatibilityRequest( + receipt=receipt, + requirements=requirements, + expected_environment=EnvironmentTuple( + os=payload.environment.os, + arch=payload.environment.arch, + ide_build=payload.environment.ide_build, + plugin_version=payload.environment.plugin_version, + host_kind=payload.environment.host_kind, + ), + expected_agent=expected_agent, + expected_protocol_version=None, + ) + return evaluate_compatibility(request), receipt_ref + + +def _compatibility_store(): + """Late import so the core research packages stay backend-agnostic.""" + from research.compatibility import store as compatibility_store + + return compatibility_store + + +class _PersistentSessionFactory: + """Bootstrap session factory that persists the authoritative session row. + + Bootstrap is the first server-side component to observe an eligible + enrollment, so it is where the durable research session is created. The + ``research_session_id`` embedded in the manifest must resolve through + :func:`research.runtime.sessions.store.get_session`; otherwise ingestion rejects + every uploaded event as ``SESSION_OUT_OF_SCOPE``. Sessions are scoped to an + opaque execution context: repeated bootstrap for the same ``(enrollment, + context_id)`` reuses that context's non-terminal session, while different + windows/projects get distinct sessions sharing the enrollment/assignment. + """ + + def __init__( + self, + db: Any, + manifest_digest_provider: Optional[Callable[[], str]] = None, + *, + commit: bool = True, + ) -> None: + self._db = db + self._manifest_digest_provider = manifest_digest_provider + # Bootstrap passes commit=False so the assignment and session commit as + # one unit of work. + self._commit = commit + + def _manifest_digest(self) -> str: + if self._manifest_digest_provider is None: + return "" + return self._manifest_digest_provider() + + def create_for_enrollment( + self, + enrollment: Any, + study: Any, + now: datetime, + context_id: str = "", + ) -> ResearchSessionRef: + # An idle-expired context session is closed here (persisted in the same + # unit of work) so the participant gets a fresh session instead of one + # the first heartbeat would end (review D-01/C-02). + existing = session_store.resolve_open_context_session( + self._db, + enrollment.enrollment_id, + context_id, + study=study, + now=now, + commit=False, + kill_switch_check=_kill_switch_for_scope( + self._db, + study_id=study.study_id, + enrollment_id=enrollment.enrollment_id, + ), + ) + if existing is not None: + return ResearchSessionRef( + research_session_id=existing.session_id, + opened_at=existing.opened_at or now, + ) + + session = ResearchSessionV1( + research_session_id=uuid.uuid4(), + enrollment_id=enrollment.enrollment_id, + study_id=study.study_id, + context_id=context_id, + state=SessionState.NOT_STARTED, + opened_at=now, + manifest_digest=self._manifest_digest(), + ) + row = session_store.create_session(self._db, session, commit=self._commit) + return ResearchSessionRef( + research_session_id=row.session_id, + opened_at=row.opened_at or now, + ) + + +@router.post( + "/research-sessions", + summary="Issue a signed bootstrap manifest for an active enrollment", +) +def create_research_session( + payload: ResearchSessionRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Validate eligibility/compatibility, then sign a secret-free manifest.""" + signer = _require_signer() + db = app.get_db_session() + try: + enrollment = _owned_enrollment(db, current_user, payload.enrollment_id) + + study = db.get(StudyRow, enrollment.study_id) + if study is None: + raise HTTPException(status_code=404, detail="Study not found") + # Only a study whose owner may change an arm by hand needs the share + # lock that serializes a first session with that change. + assignment_row = assignment_store.get_assignment_for_enrollment( + db, + enrollment.enrollment_id, + lock=manual_override_enabled(getattr(study, "research_config_json", None)), + ) + existing = ( + assignment_store.row_to_assignment(assignment_row) + if assignment_row is not None + else None + ) + if existing is not None: + allocation = allocate(enrollment, [], existing=existing, now=_now()) + else: + from sqlalchemy import select + from database.research_schemas import StudyAgentProfile + + profile_rows = db.execute( + select(StudyAgentProfile) + .where(StudyAgentProfile.study_id == enrollment.study_id) + .order_by(StudyAgentProfile.selection_order.asc()) + ).scalars().all() + profiles = [ + StudyProfileSelection( + study_id=row.study_id, + agent_profile_id=row.profile_id, + profile_digest=row.profile_digest, + profile_snapshot_json=row.profile_snapshot_json, + selection_order=row.selection_order, + ) + for row in profile_rows + ] + allocation = allocate( + enrollment, + profiles, + now=_now(), + strategy=assignment_strategy(getattr(study, "research_config_json", None)), + ) + if allocation.outcome not in ( + AllocationOutcome.CREATED, + AllocationOutcome.EXISTING, + ): + raise HTTPException( + status_code=409, detail=_issue_payload(allocation.issue) + ) + assert allocation.assignment is not None + + release_id = allocation.assignment.profile_snapshot_json.get("release_id") + release = None + if release_id: + release_row = registry_store.get_release(db, release_id) + if release_row is not None: + release = registry_store.row_to_release(release_row) + + compatibility_result, receipt_ref = _evaluate_compatibility(payload, release, db) + + # Single unit of work (lock order: participant/enrollment already + # resolved by the caller → sticky assignment → execution session when + # requested). The session factory is commit=False, so every row created + # here lands only in the one commit below; any failure rolls back. + result = compose_bootstrap( + enrollment, + study, + allocation.assignment, + release, + receipt_ref, + _PersistentSessionFactory(db, commit=False), + signer, + now=_now(), + compatibility_result=compatibility_result, + platform=(payload.environment.os, payload.environment.arch), + kill_switch_check=_kill_switch_for_scope( + db, + study_id=study.study_id, + enrollment_id=enrollment.enrollment_id, + ), + context_id=payload.context_id, + prepare_only=payload.prepare_only, + plugin_version=payload.environment.plugin_version, + ) + if result.manifest is None and result.preparation is None: + # Nothing is committed yet: a failure here (kill switch, incompatible + # environment, unresolved release) leaves no orphan session or + # half-assignment. + db.rollback() + raise HTTPException( + status_code=409, detail=_issue_payload(result.issue) + ) + + manifest = result.manifest + # Persist the sticky assignment after preparation or bootstrap succeeds, + # still inside the same transaction. A concurrent first use that + # won the unique constraint is a retryable conflict (nothing committed). + if allocation.created: + winner_row = assignment_store.create_assignment( + db, allocation.assignment, commit=False + ) + if ( + winner_row is not None + and winner_row.assignment_id != allocation.assignment.assignment_id + ): + db.rollback() + raise HTTPException( + status_code=409, + detail={ + "code": "ASSIGNMENT_RACE", + "message": ( + "the sticky assignment was allocated concurrently; " + "retry to adopt the existing profile" + ), + }, + ) + # Single commit for assignment + session. + db.commit() + + if result.preparation is not None: + return JsonResponseWithStatus( + status_code=200, + content={"preparation": result.preparation.model_dump(mode="json")}, + ) + + # The session capability is verified statelessly from its HMAC signature + # and the enrollment's revocation epoch, so it is never persisted. + return JsonResponseWithStatus( + status_code=201, + content={ + "manifest": manifest.model_dump(mode="json"), + "manifest_digest": manifest.manifest_digest, + "signature": manifest.signature, + }, + ) + finally: + db.close() diff --git a/src/backend/routers/research/budgets.py b/src/backend/routers/research/budgets.py new file mode 100644 index 0000000..3a80304 --- /dev/null +++ b/src/backend/routers/research/budgets.py @@ -0,0 +1,543 @@ +"""Participant budgets for studies on the shared provider key. + +Mounted under ``/api/research/studies``. Study owners and administrators +manage the study default, apply it to participants still on the default, and +adjust (top up / set) individual limits with an audited reason and an +idempotency key. All amounts travel as decimal USD strings at the boundary and +are stored as integer micro-USD; the ledger (``research.budget.ledger``) owns +every mutation, so the limit is enforced exactly once, at reservation time. +""" + +from __future__ import annotations + +import re +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from decimal import Decimal +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, ConfigDict, Field + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import AuthenticatedUser, get_current_user +from backend.routers.research.access import require_study_owner +from database.db_schemas import Study as StudyRow +from database.research_schemas import ( + ADJUSTMENT_APPLY_DEFAULT, + ADJUSTMENT_SET_LIMIT, + ADJUSTMENT_TOP_UP, + LIMIT_SOURCE_ADJUSTED, +) +from research.analysis.study_analytics import store as analytics_store +from research.budget import ledger, study_policy +from research.budget.errors import ( + AdjustmentInvalid, + BalanceMissing, + IdempotencyConflict, +) +from research.budget.pricing import MICRO_USD_PER_USD, parse_usd_amount, usd_to_micro +from research.study.protocol import store as study_store + +router = APIRouter() + +_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9_-]{8,128}$") + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _detail(code: str, message: str, **extra: Any) -> dict[str, Any]: + return {"code": code, "message": message, **extra} + + +def _authorize(db: Any, current_user: AuthenticatedUser, study_id: uuid.UUID): + """404 for an unknown study, 403 ``FORBIDDEN_STUDY`` for a non-owner.""" + study = study_store.get_study(db, study_id) + if study is None: + raise HTTPException(status_code=404, detail="Study not found") + require_study_owner(current_user, study) + return db.get(StudyRow, study_id) + + +def _require_enrollment(db: Any, study_id: uuid.UUID, enrollment_id: uuid.UUID) -> None: + if analytics_store.enrollment_study_id(db, enrollment_id) != study_id: + raise HTTPException( + status_code=404, + detail=_detail("ENROLLMENT_NOT_FOUND", "no such enrollment in this study"), + ) + + +def _stopped(study: Any) -> bool: + return getattr(study, "research_status", None) == "STUDY_STOPPED" + + +def _usd(micro: Optional[int]) -> Optional[str]: + return study_policy.micro_to_usd_string(micro) + + +def _amount(value: Optional[str], *, field: str = "amount_usd") -> int: + try: + return usd_to_micro(parse_usd_amount(value)) + except ValueError as error: + raise HTTPException( + status_code=422, + detail=_detail("BUDGET_INVALID", str(error), field=field), + ) from error + + +def _reason(value: Optional[str]) -> str: + text = (value or "").strip() + if not text or len(text) > 500: + raise HTTPException( + status_code=422, + detail=_detail( + "REASON_REQUIRED", "a reason of 1-500 characters is required", field="reason" + ), + ) + return text + + +def _idempotency_key(value: Optional[str]) -> str: + text = (value or "").strip() + if not _IDEMPOTENCY_KEY.match(text): + raise HTTPException( + status_code=422, + detail=_detail( + "IDEMPOTENCY_KEY_INVALID", + "idempotency_key must be 8-128 characters of [A-Za-z0-9_-]", + field="idempotency_key", + ), + ) + return text + + +def _adjustment_payload(record) -> dict[str, Any]: + return { + "adjustment_id": str(record.adjustment_id), + "enrollment_id": str(record.enrollment_id), + "kind": record.kind, + "delta_micro_usd": record.delta_micro_usd, + "delta_usd": _usd(record.delta_micro_usd), + "limit_before_micro_usd": record.limit_before_micro_usd, + "limit_after_micro_usd": record.limit_after_micro_usd, + "in_flight_micro_usd": record.in_flight_micro_usd, + "reason": record.reason, + "actor": record.actor, + "idempotency_key": record.idempotency_key, + "occurred_at": record.occurred_at.isoformat() if record.occurred_at else None, + } + + +def _ledger_entry_payload(row: dict[str, Any]) -> dict[str, Any]: + def iso(value: Any) -> Any: + return value.isoformat() if isinstance(value, datetime) else value + + return { + "reservation_id": str(row["reservation_id"]), + "model": row["model"], + "entry_point": row["entry_point"], + "request_id": row["request_id"], + "research_session_id": ( + str(row["research_session_id"]) if row["research_session_id"] else None + ), + "state": row["state"], + "hold_micro_usd": row["hold_micro_usd"], + "charged_micro_usd": row["charged_micro_usd"], + "estimated_prompt_tokens": row["estimated_prompt_tokens"], + "output_cap_tokens": row["output_cap_tokens"], + "prompt_tokens": row["prompt_tokens"], + "completion_tokens": row["completion_tokens"], + "cached_prompt_tokens": row["cached_prompt_tokens"], + "usage_source": row["usage_source"], + "resolution_reason": row["resolution_reason"], + "upstream_status": row["upstream_status"], + "finish_reason": row["finish_reason"], + "reserved_at": iso(row["reserved_at"]), + "deadline_at": iso(row["deadline_at"]), + "resolved_at": iso(row["resolved_at"]), + } + + +def _cursor(value: Optional[str]) -> Optional[datetime]: + if value is None or not value.strip(): + return None + try: + parsed = datetime.fromisoformat(value.strip()) + except ValueError as error: + raise HTTPException( + status_code=422, detail=_detail("INVALID_CURSOR", "cursor must be an ISO timestamp") + ) from error + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed + + +# --------------------------------------------------------------------------- +# Study-level policy +# --------------------------------------------------------------------------- + + +class StudyBudgetUpdateRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + default_budget_usd: Optional[str] = None + warning_fraction: Optional[float] = Field(default=None, gt=0, le=1) + + +class ApplyDefaultRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + reason: str + idempotency_key: str + + +def _study_budget_payload(db: Any, study: Any) -> dict[str, Any]: + selections = study_policy.metered_selections(db, study.study_id) + policy = study_policy.budget_policy_payload(study, selections) + default = policy["default_budget_micro_usd"] + balances = ledger.list_study_balances(db, study.study_id) + on_default = [b for b in balances if b.limit_source != LIMIT_SOURCE_ADJUSTED] + spend = ledger.study_spend_summary(db, study.study_id) + missing = study_policy.missing_prices(db, selections) + return { + "study_id": str(study.study_id), + **policy, + "editable": not _stopped(study), + "participants": { + "total": len(balances), + "on_default": sum(1 for b in on_default if b.limit_micro_usd == default), + "on_old_default": sum(1 for b in on_default if b.limit_micro_usd != default), + "custom": sum(1 for b in balances if b.limit_source == LIMIT_SOURCE_ADJUSTED), + "exhausted": sum(1 for b in balances if b.exhausted), + }, + "metered_spend_micro_usd": spend["metered_spend_micro_usd"], + "metered_spend_usd": _usd(spend["metered_spend_micro_usd"]), + "metered_calls": spend["metered_calls"], + "reserved_micro_usd": spend["reserved_micro_usd"], + "pricing": {"complete": not missing, "missing": missing}, + } + + +@router.get("/{study_id}/budget", summary="Study budget policy and totals (owner/admin)") +def get_study_budget( + study_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + study = _authorize(db, current_user, study_id) + return JsonResponseWithStatus(status_code=200, content=_study_budget_payload(db, study)) + finally: + db.close() + + +@router.patch("/{study_id}/budget", summary="Update the default participant budget (owner/admin)") +def update_study_budget( + study_id: uuid.UUID, + payload: StudyBudgetUpdateRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + study = _authorize(db, current_user, study_id) + if _stopped(study): + raise HTTPException( + status_code=409, detail=_detail("STUDY_STOPPED", "the study has been stopped") + ) + if not study_policy.metered_selections(db, study_id): + raise HTTPException( + status_code=409, + detail=_detail( + "STUDY_NOT_METERED", + "no selected profile runs Goose or the built-in agent; budgets do not apply", + ), + ) + if payload.default_budget_usd is None and payload.warning_fraction is None: + raise HTTPException( + status_code=422, + detail=_detail("BUDGET_INVALID", "nothing to update"), + ) + if payload.default_budget_usd is not None: + study.inference_budget_default_micro_usd = _amount( + payload.default_budget_usd, field="default_budget_usd" + ) + if payload.warning_fraction is not None: + study.inference_budget_warning_fraction = Decimal(str(round(payload.warning_fraction, 3))) + study.inference_budget_updated_at = _now() + study.inference_budget_updated_by = current_user.email + db.add(study) + db.commit() + db.refresh(study) + return JsonResponseWithStatus(status_code=200, content=_study_budget_payload(db, study)) + finally: + db.close() + + +@router.post( + "/{study_id}/budget/apply-default", + summary="Apply the current default to participants still on the old default (owner/admin)", +) +def apply_default_budget( + study_id: uuid.UUID, + payload: ApplyDefaultRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + study = _authorize(db, current_user, study_id) + if _stopped(study): + raise HTTPException( + status_code=409, detail=_detail("STUDY_STOPPED", "the study has been stopped") + ) + if not study_policy.metered_selections(db, study_id): + raise HTTPException( + status_code=409, + detail=_detail("STUDY_NOT_METERED", "budgets do not apply to this study"), + ) + reason = _reason(payload.reason) + key = _idempotency_key(payload.idempotency_key) + default = int(getattr(study, "inference_budget_default_micro_usd", 0) or 0) + applied = skipped = 0 + for balance in ledger.list_study_balances(db, study_id): + if balance.limit_source == LIMIT_SOURCE_ADJUSTED or balance.limit_micro_usd == default: + skipped += 1 + continue + try: + record = ledger.adjust( + db, + enrollment_id=balance.enrollment_id, + kind=ADJUSTMENT_APPLY_DEFAULT, + amount_micro_usd=default, + actor=current_user.email, + reason=reason, + idempotency_key=f"{key}:{balance.enrollment_id}", + ) + except IdempotencyConflict as error: + raise HTTPException( + status_code=409, + detail=_detail("IDEMPOTENCY_KEY_REUSED", str(error)), + ) from error + if record is None: + skipped += 1 + else: + applied += 1 + return JsonResponseWithStatus( + status_code=200, + content={ + "applied": applied, + "skipped": skipped, + "default_budget_micro_usd": default, + "default_budget_usd": _usd(default), + }, + ) + finally: + db.close() + + +# --------------------------------------------------------------------------- +# Per-enrollment balance, adjustments and ledger +# --------------------------------------------------------------------------- + + +class AdjustmentRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + kind: str + amount_usd: str + reason: str + idempotency_key: str + + +def _enrollment_budget_payload(db: Any, study: Any, enrollment_id: uuid.UUID) -> dict[str, Any]: + view = ledger.balance_view(db, enrollment_id) + warning_fraction = float(getattr(study, "inference_budget_warning_fraction", None) or 0.8) + if view is None: + return { + "enrollment_id": str(enrollment_id), + "metered": False, + "balance": None, + "ledger_summary": None, + "recent_adjustments": [], + } + entries = ledger.list_reservations(db, enrollment_id, limit=200) + settled_states = {"SETTLED", "FORFEITED", "EXPIRED"} + return { + "enrollment_id": str(enrollment_id), + "metered": True, + "balance": { + **ledger.balance_summary(view), + "limit_usd": _usd(view.limit_micro_usd), + "consumed_usd": _usd(view.settled_micro_usd), + "remaining_usd": _usd(max(view.available_micro_usd, 0)), + "warning_fraction": warning_fraction, + "settled_prompt_tokens": view.settled_prompt_tokens, + "settled_completion_tokens": view.settled_completion_tokens, + }, + "ledger_summary": { + "calls": len(entries), + "settled_micro_usd": sum( + int(e["charged_micro_usd"] or 0) for e in entries if e["state"] == "SETTLED" + ), + "forfeited_micro_usd": sum( + int(e["charged_micro_usd"] or 0) + for e in entries + if e["state"] in {"FORFEITED", "EXPIRED"} + ), + "voided": sum(1 for e in entries if e["state"] == "VOIDED"), + "open": sum(1 for e in entries if e["state"] == "RESERVED"), + "charged_micro_usd": sum( + int(e["charged_micro_usd"] or 0) for e in entries if e["state"] in settled_states + ), + "last_call_at": entries[0]["reserved_at"].isoformat() if entries else None, + }, + "recent_adjustments": [ + _adjustment_payload(item) for item in ledger.list_adjustments(db, enrollment_id, limit=10) + ], + } + + +@router.get( + "/{study_id}/enrollments/{enrollment_id}/budget", + summary="One participant's budget balance (owner/admin)", +) +def get_enrollment_budget( + study_id: uuid.UUID, + enrollment_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + study = _authorize(db, current_user, study_id) + _require_enrollment(db, study_id, enrollment_id) + return JsonResponseWithStatus( + status_code=200, content=_enrollment_budget_payload(db, study, enrollment_id) + ) + finally: + db.close() + + +@router.post( + "/{study_id}/enrollments/{enrollment_id}/budget/adjustments", + summary="Top up or set one participant's budget (owner/admin)", +) +def adjust_enrollment_budget( + study_id: uuid.UUID, + enrollment_id: uuid.UUID, + payload: AdjustmentRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + study = _authorize(db, current_user, study_id) + _require_enrollment(db, study_id, enrollment_id) + if _stopped(study): + raise HTTPException( + status_code=409, detail=_detail("STUDY_STOPPED", "the study has been stopped") + ) + kind = (payload.kind or "").strip().upper() + if kind not in (ADJUSTMENT_TOP_UP, ADJUSTMENT_SET_LIMIT): + raise HTTPException( + status_code=422, + detail=_detail("BUDGET_INVALID", "kind must be TOP_UP or SET_LIMIT", field="kind"), + ) + amount = _amount(payload.amount_usd) + reason = _reason(payload.reason) + key = _idempotency_key(payload.idempotency_key) + if ledger.balance_view(db, enrollment_id) is None: + raise HTTPException( + status_code=409, + detail=_detail("ENROLLMENT_NOT_METERED", "this participant has no budget balance"), + ) + try: + record = ledger.adjust( + db, + enrollment_id=enrollment_id, + kind=kind, + amount_micro_usd=amount, + actor=current_user.email, + reason=reason, + idempotency_key=key, + ) + except IdempotencyConflict as error: + raise HTTPException( + status_code=409, detail=_detail("IDEMPOTENCY_KEY_REUSED", str(error)) + ) from error + except AdjustmentInvalid as error: + raise HTTPException( + status_code=422, detail=_detail("BUDGET_INVALID", str(error)) + ) from error + except BalanceMissing as error: + raise HTTPException( + status_code=409, detail=_detail("ENROLLMENT_NOT_METERED", str(error)) + ) from error + body = _enrollment_budget_payload(db, study, enrollment_id) + return JsonResponseWithStatus( + status_code=200 if record.replayed else 201, + content={ + "adjustment": _adjustment_payload(record), + "replayed": bool(record.replayed), + "balance": body["balance"], + }, + ) + finally: + db.close() + + +@router.get( + "/{study_id}/enrollments/{enrollment_id}/budget/ledger", + summary="One participant's metered calls, newest first (owner/admin)", +) +def get_enrollment_ledger( + study_id: uuid.UUID, + enrollment_id: uuid.UUID, + limit: int = Query(50, ge=1, le=200), + cursor: Optional[str] = Query(None), + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + _authorize(db, current_user, study_id) + _require_enrollment(db, study_id, enrollment_id) + rows = ledger.list_reservations(db, enrollment_id, limit=limit, before=_cursor(cursor)) + entries = [_ledger_entry_payload(row) for row in rows] + next_cursor = entries[-1]["reserved_at"] if len(entries) == limit else None + return JsonResponseWithStatus( + status_code=200, content={"entries": entries, "next_cursor": next_cursor} + ) + finally: + db.close() + + +@router.get( + "/{study_id}/enrollments/{enrollment_id}/budget/adjustments", + summary="One participant's budget adjustments, newest first (owner/admin)", +) +def get_enrollment_adjustments( + study_id: uuid.UUID, + enrollment_id: uuid.UUID, + limit: int = Query(50, ge=1, le=200), + cursor: Optional[str] = Query(None), + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + _authorize(db, current_user, study_id) + _require_enrollment(db, study_id, enrollment_id) + records = ledger.list_adjustments(db, enrollment_id, limit=limit, before=_cursor(cursor)) + items = [_adjustment_payload(record) for record in records] + next_cursor = items[-1]["occurred_at"] if len(items) == limit else None + return JsonResponseWithStatus( + status_code=200, content={"adjustments": items, "next_cursor": next_cursor} + ) + finally: + db.close() diff --git a/src/backend/routers/research/inference.py b/src/backend/routers/research/inference.py new file mode 100644 index 0000000..74d4abe --- /dev/null +++ b/src/backend/routers/research/inference.py @@ -0,0 +1,266 @@ +"""The research inference gateway: Goose's metered path to the study's provider key. + +``POST /api/research/inference/v1/chat/completions`` is an OpenAI-compatible +Chat Completions endpoint. Goose is pointed at it through the release's +runtime bindings (``OPENAI_HOST``/``OPENAI_BASE_PATH``) and presents the +inference capability minted at bootstrap as its bearer token. Every call: + +1. decodes and verifies the capability (signature, audience ``inference``, + scope ``inference:relay``, expiry, revocation epoch, enrollment/study); +2. re-checks the funded gate (ACTIVE enrollment, study open, kill switch at + study and enrollment scope); +3. resolves the sticky assignment's frozen profile (Goose only) and the + funding owner's provider connection; +4. runs ``run_inference`` with a budget meter: worst-case hold reserved before + the request leaves, ``max_tokens`` capped, actual usage settled afterwards. + +The model is forced to the frozen profile's. Goose's tool definitions pass +through untouched (``tool_filtering=False``) unless the pinned release declares +its ``tools`` binding with the ``gateway`` transport: Goose reads no tool +setting from its environment, so the gateway is then the one enforcement point: +the model is offered only the frozen selection (an empty one means no tools), +and a call it still makes to a withheld tool is removed from the response +before Goose can run it. No ``agent_task`` is materialised +and no observation telemetry is written here: Goose's telemetry keeps coming +from the ACP proxy, and the ledger row is the cost record. The research +session is deliberately not required to be live (budgets are per enrollment; +an idle-ended session must not break a running chat); the reservation is +attributed to the enrollment's active session when there is one. +""" + +from __future__ import annotations + +import json +import logging +import os +import uuid +from dataclasses import dataclass +from typing import Optional + +from fastapi import APIRouter, Body, Depends, Header, HTTPException, Response +from fastapi.concurrency import run_in_threadpool + +from agents import inference, registry +from agents import provider as provider_module +from App import App +from backend.routers.research import access +from research.budget import ( + InferenceMeter, + decode_capability_bearer, + verify_inference_capability, +) +from research.participants import identity as identity_store +from research.runtime.sessions import store as session_store +from research.study.agents import store as agents_store +from research.study.agents.distributions import release_enforces_tools_at_gateway +from research.study.agents.enums import INFERENCE_GATEWAY_FRAMEWORKS + +router = APIRouter() + +CHAT_COMPLETIONS_PATH = "/v1/chat/completions" + + +def _signing_secret() -> str: + return os.environ.get("BOOTSTRAP_SIGNING_SECRET") or "" + + +@dataclass(frozen=True) +class GatewayContext: + enrollment_id: uuid.UUID + study_id: uuid.UUID + research_session_id: Optional[uuid.UUID] + profile: object + connection: provider_module.ResolvedConnection + #: The frozen tool selection enforced here, or ``None`` to pass tools through. + tool_allowlist: Optional[frozenset[str]] = None + + +def _gateway_tool_allowlist(db, profile) -> Optional[frozenset[str]]: + """The profile's frozen tool selection when its release enforces it here.""" + release_id = getattr(profile, "release_id", None) + release = agents_store.get_release(db, release_id) if release_id else None + if release is None or not release_enforces_tools_at_gateway(release): + return None + try: + parsed = json.loads(getattr(profile, "tools_json", None) or "[]") + except (TypeError, ValueError): + parsed = None + if not isinstance(parsed, list): + # A selection that cannot be read is never widened to every tool. + return frozenset() + return frozenset(str(item) for item in parsed) + + +def _authorize(app: App, capability) -> GatewayContext: + """Verify the capability and the funded gate; resolve profile + connection.""" + db = app.get_db_session() + try: + enrollment_row = identity_store.get_enrollment(db, capability.enrollment_id) + if enrollment_row is None: + raise HTTPException( + status_code=401, + detail={"code": "CAPABILITY_INVALID", "message": "unknown enrollment"}, + ) + verification = verify_inference_capability( + capability, + secret=_signing_secret(), + current_revocation_epoch=int(enrollment_row.revocation_epoch or 0), + expected_enrollment_id=enrollment_row.enrollment_id, + expected_study_id=enrollment_row.study_id, + ) + if not verification.ok: + raise HTTPException( + status_code=401, + detail={ + "code": "CAPABILITY_INVALID", + "reason": verification.reason.value, + "message": verification.message, + }, + ) + participant = identity_store.get_participant(db, enrollment_row.participant_id) + if participant is None: + raise HTTPException( + status_code=401, + detail={"code": "CAPABILITY_INVALID", "message": "unknown participant"}, + ) + try: + active = access.require_funded_access( + db, account_id=participant.account_id, study_id=enrollment_row.study_id + ) + except access.FundedAccessRefused as exc: + raise HTTPException( + status_code=403, detail={"code": exc.code, "message": exc.message} + ) from exc + if active.enrollment_id != enrollment_row.enrollment_id: + raise HTTPException( + status_code=403, + detail={ + "code": "ENROLLMENT_NOT_ACTIVE", + "message": "the capability's enrollment is no longer the live one", + }, + ) + assignment = registry.resolve_assignment_context(db, participant.account_id) + if assignment is None: + raise HTTPException( + status_code=503, + detail={ + "code": "ASSIGNMENT_UNAVAILABLE", + "message": "no assigned agent profile could be resolved", + }, + ) + profile = assignment.profile + if profile.framework_version not in INFERENCE_GATEWAY_FRAMEWORKS: + raise HTTPException( + status_code=403, + detail={ + "code": "RUNTIME_NOT_SUPPORTED", + "message": ( + f"runtime {profile.framework_version!r} does not use the " + "research inference gateway" + ), + }, + ) + funding_owner_user_id, owner_is_admin = provider_module.funding_owner_for_profile( + db, profile + ) + try: + connection = provider_module.resolve_task_connection( + db, profile, funding_owner_user_id, owner_is_admin=owner_is_admin + ) + except provider_module.ProviderReadinessError as exc: + raise HTTPException( + status_code=503, detail={"code": exc.code, "message": exc.message} + ) from exc + session_row = session_store.get_active_session_for_enrollment( + db, enrollment_row.enrollment_id + ) + research_session_id = ( + getattr(session_row, "session_id", None) if session_row is not None else None + ) + return GatewayContext( + enrollment_id=enrollment_row.enrollment_id, + study_id=enrollment_row.study_id, + research_session_id=research_session_id or capability.research_session_id, + profile=profile, + connection=connection, + tool_allowlist=_gateway_tool_allowlist(db, profile), + ) + finally: + db.close() + + +@router.post(CHAT_COMPLETIONS_PATH) +async def research_gateway_chat_completions( + body: dict = Body(...), + authorization: str = Header(default=""), + app: App = Depends(App.get_instance), +) -> Response: + """Metered, server-keyed Chat Completions for a Goose study arm.""" + capability = decode_capability_bearer(authorization) + if capability is None: + raise HTTPException( + status_code=401, + detail={ + "code": "CAPABILITY_INVALID", + "message": "the Authorization bearer is not a research inference capability", + }, + ) + if "input" in body or not isinstance(body.get("messages"), list): + raise HTTPException( + status_code=400, + detail={ + "code": "CHAT_COMPLETIONS_REQUIRED", + "message": "the research inference gateway accepts Chat Completions bodies only", + }, + ) + context = await run_in_threadpool(_authorize, app, capability) + meter = InferenceMeter( + app=app, + enrollment_id=context.enrollment_id, + study_id=context.study_id, + connection_id=context.connection.connection_id, + model=context.profile.model, + entry_point="research_gateway", + research_session_id=context.research_session_id, + ) + logging.info( + "[Research/inference] enrollment=%s model=%s stream=%s messages=%d", + str(context.enrollment_id)[:8], + context.profile.model, + bool(body.get("stream")), + len(body.get("messages") or []), + ) + if context.tool_allowlist is not None: + # Before the meter: the hold is estimated on exactly what goes upstream. + requested, kept = inference.restrict_chat_completion_tools(body, context.tool_allowlist) + logging.info( + "[Research/inference] tools enforced — requested=%d kept=%d", + len(requested), + len(kept), + ) + if context.tool_allowlist and requested and not kept: + # Names match exactly: an agent build that renamed its tools would + # silently turn this arm into a no-tools arm. + logging.warning( + "[Research/inference] none of the agent's tools %s is in the frozen selection %s", + sorted(set(requested)), + sorted(context.tool_allowlist), + ) + return await inference.run_inference( + task_uuid=uuid.uuid4(), + session_uuid=context.research_session_id or uuid.uuid4(), + openai_body=body, + enrichment=None, + agent_profile=getattr(context.profile, "name", None), + model=context.profile.model, + connection=context.connection, + temperature=getattr(context.profile, "temperature", None), + framework_version=context.profile.framework_version, + profile_tools_json=None, + content_included=False, + record_observation_events=False, + tool_filtering=False, + response_tool_allowlist=context.tool_allowlist, + meter=meter, + app=app, + ) diff --git a/src/backend/routers/research/join.py b/src/backend/routers/research/join.py new file mode 100644 index 0000000..f10d29c --- /dev/null +++ b/src/backend/routers/research/join.py @@ -0,0 +1,278 @@ +"""Web-only research study join and consent.""" + +from __future__ import annotations + +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, ConfigDict, Field + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import AuthenticatedUser, get_current_user +from privacy import collection +from research.study.lifecycle import ( + AlreadyEnrolledError, + ActiveEnrollmentError, + StudyStoppedError, + open_study_enrollment, +) +from research.runtime.assignment.hashing import manual_override_enabled +from research.study import consent +from research.study.protocol import store as study_store +from research.telemetry.enums import FieldClass +from research.telemetry.privacy.engine import PrivacyPolicy + +router = APIRouter() + +# The consent notice is composed from the study's frozen policy as ingestion +# resolves it (PrivacyPolicy.from_study_policy), so it states what is stored. +GLOBAL_CONSENT_TEXT = ( + "This study collects metadata about how you use the research agent and the " + "IDE: which events happen and when (for example prompts you send, tools the " + "agent runs, approvals, errors, and files you open, edit and save), with " + "timings, counts and token usage. You are randomly assigned to one of the " + "study's agent configurations. The research agent sends your prompts and the " + "code it works with to an AI model provider to produce its responses, " + "whether or not this study stores them. Provider credentials are never " + "collected, and values that look like secrets (such as API keys and tokens) " + "are removed, although that filter only recognises common formats. Your " + "study-local pseudonym is used in research data; your account identity " + "remains private." +) + +#: The assignment sentence of ``GLOBAL_CONSENT_TEXT`` and its replacement for a +#: study whose protocol lets the owner set an arm by hand before first use. +RANDOM_ASSIGNMENT_SENTENCE = "You are randomly assigned to one of the study's agent configurations." +MANUAL_ASSIGNMENT_SENTENCE = ( + "You are assigned to one of the study's agent configurations, at random or by " + "the research team." +) + +TOOL_TITLES_CONSENT_TEXT = ( + "The metadata includes the titles of the agent's tool calls and its error " + "messages, which can contain command lines, file paths (which can include " + "your computer's user name), search terms and URLs." +) + +CODE_METADATA_CONSENT_TEXT = ( + "The metadata also includes code metadata such as file paths, file types, " + "languages and symbol names." +) + +HASHED_CODE_METADATA_CONSENT_TEXT = ( + "Code metadata such as file types and languages is stored only as hashes, " + "which can be reversed for common values." +) + +METADATA_ONLY_CONSENT_TEXT = ( + "This study is configured for metadata only: the text of your prompts, the " + "agent's responses and reasoning, tool arguments and results, and file " + "contents are not stored" +) + +#: Appended to the metadata-only sentence when tool titles are kept. +METADATA_ONLY_TITLES_CAVEAT = ", apart from what tool titles and error messages may contain" + +CONTENT_CAPTURE_CONSENT_TEXT = ( + "This study is configured to store content: your prompts, the agent's " + "responses and reasoning, tool arguments and results, and file contents may " + "be stored under the study's approved telemetry policy." +) + + +def consent_text(telemetry_policy: Any, *, manual_assignment: bool = False) -> str: + """The participant-facing notice for a frozen study telemetry policy.""" + raw = telemetry_policy if isinstance(telemetry_policy, dict) else {} + allowed = set( + PrivacyPolicy.from_study_policy(raw, consent_active=True).allowed_field_classes + ) + titles_kept = FieldClass.BEHAVIORAL in allowed + parts = [ + GLOBAL_CONSENT_TEXT.replace(RANDOM_ASSIGNMENT_SENTENCE, MANUAL_ASSIGNMENT_SENTENCE) + if manual_assignment + else GLOBAL_CONSENT_TEXT + ] + if titles_kept: + parts.append(TOOL_TITLES_CONSENT_TEXT) + parts.append( + CODE_METADATA_CONSENT_TEXT + if FieldClass.CODE_METADATA in allowed + else HASHED_CODE_METADATA_CONSENT_TEXT + ) + if raw.get("content_capture") is True: + parts.append(CONTENT_CAPTURE_CONSENT_TEXT) + else: + parts.append( + METADATA_ONLY_CONSENT_TEXT + + (METADATA_ONLY_TITLES_CAVEAT if titles_kept else "") + + "." + ) + return " ".join(parts) + + +def _collection_policy(study: Any) -> dict[str, Any]: + """Project the frozen study telemetry policy without exposing internals.""" + config = getattr(study, "research_config_json", None) or {} + raw = config.get("telemetry_policy") or {} + if not isinstance(raw, dict): + raw = {} + return { + "content_capture": raw.get("content_capture") is True, + "allowed_field_classes": list(raw.get("allowed_field_classes") or []), + } + + +def study_consent_view(config: Any) -> dict[str, Any]: + """The consent view of a study: its own form (if any) plus the platform notice.""" + config = config if isinstance(config, dict) else {} + notice = consent_text( + config.get("telemetry_policy"), + manual_assignment=manual_override_enabled(config), + ) + return consent.build_view(config, notice) + + +def _consent_payload(study: Any) -> dict[str, Any]: + """Render the actual frozen collection policy in participant-facing text.""" + config = getattr(study, "research_config_json", None) or {} + view = study_consent_view(config) + return { + # ``text`` stays the platform notice so older clients keep working. + "text": view["notice"], + "collection_policy": _collection_policy(study), + "custom": view["document"] is not None, + "document": view["document"], + "notice": view["notice"], + "statements": view["statements"], + "digest": consent.view_digest(view), + } + + +def _require_authenticated_user(current_user: AuthenticatedUser) -> AuthenticatedUser: + if current_user is None: + raise HTTPException(status_code=401, detail="Authentication required") + return current_user + + +class JoinRequestBody(BaseModel): + model_config = ConfigDict(extra="forbid") + + join_code: str = Field(min_length=1) + accept_consent: bool = False + # The digest of the consent view the participant reviewed, and which of its + # statements they ticked. Required for a study with its own consent form. + consent_digest: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$") + consent_statements: Optional[dict[str, bool]] = Field(default=None, max_length=20) + + +def _study_from_code(db: Any, join_code: str): + return study_store.get_study_by_join_code(db, join_code) + + +def _study_payload(study: Any) -> dict[str, Any]: + return { + "study_id": str(study.study_id), + "name": study.name, + "description": study.description, + "research_status": getattr(study, "research_status", None), + } + + +@router.get("/{join_code}", summary="Resolve a study-owned join code") +def resolve_join_code( + join_code: str, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + _require_authenticated_user(current_user) + db = app.get_db_session() + try: + study = _study_from_code(db, join_code) + if study is None: + raise HTTPException(status_code=404, detail="Join code not found") + if getattr(study, "research_status", None) == "STUDY_STOPPED": + raise HTTPException( + status_code=409, + detail={"code": "STUDY_STOPPED", "message": "the study has been stopped"}, + ) + return JsonResponseWithStatus( + status_code=200, + content={ + "join_code": study.join_code, + "study": _study_payload(study), + "consent": _consent_payload(study), + }, + ) + finally: + db.close() + + +@router.post("", summary="Accept consent and enroll in a study") +def redeem_join_code( + payload: JoinRequestBody, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + current_user = _require_authenticated_user(current_user) + if not payload.accept_consent: + raise HTTPException( + status_code=409, + detail={"code": "CONSENT_REQUIRED", "message": "accept consent before joining"}, + ) + db = app.get_db_session() + try: + # Joining and opting out serialize on the account row: this share lock is + # held until the enrollment commits, and an opt-out updates that row. + if not collection.lock_collection_allowed(db, current_user.user_id): + raise HTTPException( + status_code=409, + detail={ + "code": "DATA_COLLECTION_OPTED_OUT", + "message": "you have opted out of data collection; turn it back on to join a study", + }, + ) + study = _study_from_code(db, payload.join_code) + if study is None: + raise HTTPException(status_code=404, detail="join code not found") + try: + consent_digest, consent_snapshot = consent.accept( + study_consent_view(getattr(study, "research_config_json", None)), + payload.consent_digest, + payload.consent_statements, + ) + except consent.ConsentError as error: + raise HTTPException(status_code=error.status_code, detail=error.detail()) from error + try: + result = open_study_enrollment( + db, + current_user.user_id, + payload.join_code, + consent_digest=consent_digest, + consent_snapshot=consent_snapshot, + ) + except StudyStoppedError as error: + code = "STUDY_STOPPED" + raise HTTPException(status_code=409, detail={"code": code, "message": str(error)}) from error + except ActiveEnrollmentError as error: + code = "ACTIVE_ENROLLMENT_EXISTS" + raise HTTPException(status_code=409, detail={"code": code, "message": str(error)}) from error + except AlreadyEnrolledError as error: + code = "ALREADY_ENROLLED" + raise HTTPException(status_code=409, detail={"code": code, "message": str(error)}) from error + except ValueError as error: + raise HTTPException(status_code=404, detail=str(error)) from error + return JsonResponseWithStatus( + status_code=201 if result.created else 200, + content={ + "enrollment_id": str(result.enrollment_id), + "study_id": str(result.study_id), + "assignment_id": str(result.assignment_id) if result.assignment_id else None, + "agent_profile_id": str(result.agent_profile_id) if result.agent_profile_id else None, + "status": "ACTIVE", + "created": result.created, + "reused": result.reused, + }, + ) + finally: + db.close() diff --git a/src/backend/routers/research/operations.py b/src/backend/routers/research/operations.py new file mode 100644 index 0000000..6027d20 --- /dev/null +++ b/src/backend/routers/research/operations.py @@ -0,0 +1,167 @@ +"""Kill-switch API (Issue 13). + +These admin-only routes are merged with the Issue 12 read-model routes behind +the single ``/research/operations`` router (see +:mod:`backend.routers.research`). All handlers are thin; the domain logic lives +in :mod:`research.analysis.operations`. Every endpoint here requires an +operator/admin identity because it engages/releases an enforcement control. +""" + +from __future__ import annotations + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field, field_validator + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import AuthenticatedUser, require_admin +from research.analysis.operations import store as operations_store +from research.analysis.operations.enums import ( + KillSwitchScopeKind, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) +from research.analysis.operations.models import ( + KillSwitchRecord, + KillSwitchScope, +) + +operations_router = APIRouter() + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +# --------------------------------------------------------------------------- +# Request bodies +# --------------------------------------------------------------------------- + + +class KillSwitchEngageRequest(BaseModel): + """Engage a kill switch at a study or enrollment scope.""" + + scope_kind: KillSwitchScopeKind + scope_id: uuid.UUID + reason: str + effective_until: Optional[datetime] = None + + @field_validator("reason") + @classmethod + def validate_reason(cls, value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError("reason must not be blank") + return normalized + + +@operations_router.get("/kill-switch", summary="List kill-switch records") +def list_kill_switches( + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + records = operations_store.kill_switch_records(db) + return JsonResponseWithStatus( + status_code=200, + content={ + "records": [record.model_dump(mode="json") for record in records], + "engaged": [ + str(record.switch_id) + for record in records + if record.is_engaged(_now()) + ], + }, + ) + finally: + db.close() + + +@operations_router.post("/kill-switch", summary="Engage a kill switch") +def engage_kill_switch( + payload: KillSwitchEngageRequest, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + if payload.scope_kind == KillSwitchScopeKind.STUDY: + from research.study.protocol import store as study_store + + study = study_store.get_study(db, payload.scope_id) + if study is None: + raise HTTPException( + status_code=404, + detail={"code": "STUDY_NOT_FOUND", "message": "Study not found"}, + ) + if getattr(study, "research_status", None) == "STUDY_STOPPED": + raise HTTPException( + status_code=409, + detail={ + "code": "STUDY_STOPPED", + "message": "The study has stopped and cannot accept a kill switch", + }, + ) + record = KillSwitchRecord( + switch_id=uuid.uuid4(), + scope=KillSwitchScope( + kind=payload.scope_kind, scope_id=payload.scope_id + ), + reason=payload.reason, + actor=current_user.email, + engaged_at=_now(), + effective_until=payload.effective_until, + ) + row = operations_store.engage_kill_switch(db, record) + return JsonResponseWithStatus( + status_code=201, content={"switch_id": str(row.switch_id), "reason": row.reason} + ) + finally: + db.close() + + +@operations_router.post( + "/kill-switch/{switch_id}/release", summary="Release a kill switch" +) +def release_kill_switch_endpoint( + switch_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + existing = operations_store.get_kill_switch(db, switch_id) + if existing is None: + raise HTTPException(status_code=404, detail="Kill switch not found") + # Releasing is an operational remedy, not a reopening: a stopped study + # stays stopped, so its switch is never released (mirrors engage). + if existing.scope.kind == KillSwitchScopeKind.STUDY: + from research.study.protocol import store as study_store + + study = study_store.get_study(db, existing.scope.scope_id) + if study is not None and getattr(study, "research_status", None) == "STUDY_STOPPED": + raise HTTPException( + status_code=409, + detail={ + "code": "STUDY_STOPPED", + "message": ( + "The study has stopped; releasing its kill switch " + "cannot reopen it" + ), + }, + ) + row = operations_store.release_kill_switch(db, switch_id, released_at=_now()) + if row is None: + raise HTTPException(status_code=404, detail="Kill switch not found") + return JsonResponseWithStatus( + status_code=200, + content={ + "switch_id": str(row.switch_id), + "released_at": row.released_at.isoformat() if row.released_at else None, + }, + ) + finally: + db.close() diff --git a/src/backend/routers/research/packages.py b/src/backend/routers/research/packages.py new file mode 100644 index 0000000..c0d866e --- /dev/null +++ b/src/backend/routers/research/packages.py @@ -0,0 +1,83 @@ +"""Runtime package API (Issue 11). + +Mounted under ``/research/packages``. All endpoints are admin-only: they record +artifact/manifest evidence, so they are never exposed to participants. Handlers +are thin; the packaging domain logic lives in :mod:`research.study.packaging`. + +Package metadata cannot create or qualify a release; import verified bytes first. +""" + +from __future__ import annotations + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime + +from fastapi import APIRouter, Depends, HTTPException + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import AuthenticatedUser, require_admin +from research.study.agents import store as agents_store +from research.study.packaging import store as packaging_store +from research.study.packaging.models import ( + RuntimeManifestV2, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) + +packages_router = APIRouter() + + +@packages_router.get("", summary="List registered runtime packages") +def list_packages( + release_id: str | None = None, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + rows = packaging_store.list_packages(db, release_id) + return JsonResponseWithStatus( + status_code=200, + content={"packages": [packaging_store.package_summary(row) for row in rows]}, + ) + finally: + db.close() + + +@packages_router.post("", summary="Register a runtime package manifest") +def register_package( + payload: RuntimeManifestV2, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + if not payload.digest_matches(): + raise HTTPException( + status_code=422, + detail="manifest_digest does not match the canonical manifest content", + ) + if agents_store.get_release(db, payload.release_id) is None: + raise HTTPException(status_code=404, detail="Import the verified release first") + row = packaging_store.upsert_package(db, payload) + return JsonResponseWithStatus( + status_code=201, content=packaging_store.package_summary(row) + ) + finally: + db.close() + + +@packages_router.get("/{package_id}", summary="Get a runtime package") +def get_package( + package_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + row = packaging_store.get_package(db, package_id) + if row is None: + raise HTTPException(status_code=404, detail="Runtime package not found") + return JsonResponseWithStatus( + status_code=200, content=packaging_store.package_summary(row) + ) + finally: + db.close() diff --git a/src/backend/routers/research/participants.py b/src/backend/routers/research/participants.py new file mode 100644 index 0000000..ab627ed --- /dev/null +++ b/src/backend/routers/research/participants.py @@ -0,0 +1,280 @@ +"""Participant enrollment inspection and privacy-operator retention views. + +Mounted under ``/api/research/participants``. Consent is a single acceptance +recorded at join; there is no consent, re-consent or withdrawal surface here. + +* ``GET /me`` — the caller's study-local enrollment projections. +* ``GET /enrollments/{id}`` — administrator inspection. +* deletion-ledger / retention-job routes — administrator privacy-operator views + for the study-end retention path. +""" + +from __future__ import annotations + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, + require_admin, +) +from backend.routers.research.join import ( + _collection_policy as consent_collection_policy, +) +from research.participants import identity as store +from research.participants import retention as ret_api +from research.participants.enums import RetentionJobState + +router = APIRouter() + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _iso(value: Any) -> Optional[str]: + return value.isoformat() if isinstance(value, datetime) else None + + +def _my_study_payload(study_row: Any) -> Optional[dict[str, Any]]: + """The public study facts a participant saw when joining, or ``None``. + + ``collection`` is the frozen policy the join consent text is composed from + (``join.consent_text`` resolves it the way ingestion does). Participants who + joined before that text listed tool titles, error messages and code + metadata consented to the earlier, shorter notice. + """ + if study_row is None: + return None + return { + "study_id": str(study_row.study_id), + "name": study_row.name, + "description": study_row.description, + "research_status": getattr(study_row, "research_status", None), + "starts_at": _iso(getattr(study_row, "starts_at", None)), + "ends_at": _iso(getattr(study_row, "ends_at", None)), + "collection": consent_collection_policy(study_row), + } + + +def _my_enrollment_payload( + enrollment: Any, details: dict[str, Any], row: Any = None +) -> dict[str, Any]: + """The caller's own enrollment: projection fields plus study context. + + Additive over :func:`store.researcher_projection` (the plugin parses + ``enrollment_id``/``study_id``/``status``). ``runtime`` names only the + runtime kind; the assigned profile, model and digest are never returned, + so the participant stays blind to their arm. + """ + payload = store.researcher_projection(enrollment) + payload["consent_accepted_at"] = _iso(enrollment.consent_accepted_at) + payload["study"] = _my_study_payload(details.get("study_row")) + payload["runtime"] = details.get("runtime") + payload["sessions"] = details["sessions"] + payload["activity"] = details["activity"] + # Budget numbers only (no model, price or profile): ``null`` when the arm + # is not metered or no balance exists yet. + payload["budget"] = details.get("budget") + # The consent the participant accepted (document, platform notice, + # statements and their answers); ``null`` for enrollments created before + # consent versions were recorded. + snapshot = getattr(row, "consent_snapshot_json", None) + payload["consent"] = ( + {"digest": getattr(row, "consent_digest", None), **snapshot} + if isinstance(snapshot, dict) + else None + ) + return payload + + +def _issue_payload(issue: Any) -> dict[str, Any]: + if issue is None: + return {} + return issue.model_dump(mode="json") + + +def _owned_enrollment( + db: Any, current_user: AuthenticatedUser, enrollment_id: uuid.UUID +): + """Return the caller's enrollment or raise 404 without revealing ownership.""" + participant_row = store.get_participant_by_account(db, current_user.user_id) + if participant_row is None: + raise HTTPException(status_code=404, detail="Enrollment not found") + row = store.get_enrollment(db, enrollment_id) + if row is None: + raise HTTPException(status_code=404, detail="Enrollment not found") + enrollment = store.row_to_enrollment(row) + if enrollment.participant_id != participant_row.participant_id: + raise HTTPException(status_code=404, detail="Enrollment not found") + return enrollment + + +@router.get("/me", summary="Get my study enrollment status") +def get_my_status( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Return the caller's own enrollments with their study context. + + Each entry keeps the study-local projection and adds + ``consent_accepted_at``, the public ``study`` facts (with the frozen + collection policy), the ``runtime`` kind, and ``sessions``/``activity`` + summaries. Only the caller's own enrollments are read. + """ + db = app.get_db_session() + try: + participant_row = store.get_participant_by_account(db, current_user.user_id) + if participant_row is None: + return JsonResponseWithStatus(status_code=200, content={"enrollments": []}) + rows = store.list_enrollments(db, participant_row.participant_id) + rows_by_id = {row.enrollment_id: row for row in rows} + enrollments = [store.row_to_enrollment(row) for row in rows] + details = store.participant_enrollment_details(db, enrollments) + return JsonResponseWithStatus( + status_code=200, + content={ + "enrollments": [ + _my_enrollment_payload( + enrollment, + details[enrollment.enrollment_id], + rows_by_id.get(enrollment.enrollment_id), + ) + for enrollment in enrollments + ] + }, + ) + finally: + db.close() + + +@router.get("/enrollments/{enrollment_id}", summary="Inspect an enrollment") +def get_enrollment_route( + enrollment_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Administrator view of one pseudonymous enrollment projection.""" + require_admin(current_user) + db = app.get_db_session() + try: + row = store.get_enrollment(db, enrollment_id) + if row is None: + raise HTTPException(status_code=404, detail="Enrollment not found") + return JsonResponseWithStatus( + status_code=200, + content={"enrollment": store.researcher_projection(store.row_to_enrollment(row))}, + ) + finally: + db.close() + + +@router.get("/deletion-ledger", summary="Read the deletion ledger") +def read_deletion_ledger( + enrollment_id: Optional[uuid.UUID] = None, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Privacy-operator view of the append-only deletion ledger.""" + require_admin(current_user) + db = app.get_db_session() + try: + rows = store.list_deletion_ledger(db, enrollment_id) + return JsonResponseWithStatus( + status_code=200, + content={"entries": [store.deletion_ledger_summary(row) for row in rows]}, + ) + finally: + db.close() + + +# --------------------------------------------------------------------------- +# Retention execution jobs (privacy-operator) +# --------------------------------------------------------------------------- + +_RETRYABLE_STATES = frozenset({RetentionJobState.FAILED, RetentionJobState.RETRYABLE}) + + +@router.get("/retention-jobs/{job_id}", summary="Fetch one retention job's status") +def get_retention_job_status( + job_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Privacy-operator view of one content-free retention job summary.""" + require_admin(current_user) + db = app.get_db_session() + try: + row = ret_api.get_retention_job(db, job_id) + if row is None: + raise HTTPException(status_code=404, detail="Retention job not found") + return JsonResponseWithStatus( + status_code=200, + content={"job": ret_api.job_summary(ret_api.row_to_retention_job(row))}, + ) + finally: + db.close() + + +@router.get( + "/enrollments/{enrollment_id}/retention-jobs", + summary="List an enrollment's retention jobs", +) +def list_enrollment_retention_jobs( + enrollment_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Privacy-operator view of an enrollment's retention-job history.""" + require_admin(current_user) + db = app.get_db_session() + try: + rows = ret_api.list_retention_jobs_for_enrollment(db, enrollment_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "jobs": [ + ret_api.job_summary(ret_api.row_to_retention_job(row)) + for row in rows + ] + }, + ) + finally: + db.close() + + +@router.post( + "/retention-jobs/{job_id}/retry", + summary="Re-drive a FAILED/RETRYABLE retention job", +) +def retry_retention_job( + job_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Re-drive a failed/retryable job; a completed job is never re-executed.""" + require_admin(current_user) + db = app.get_db_session() + try: + row = ret_api.get_retention_job(db, job_id) + if row is None: + raise HTTPException(status_code=404, detail="Retention job not found") + job = ret_api.row_to_retention_job(row) + if job.state not in _RETRYABLE_STATES: + raise HTTPException( + status_code=409, + detail={"code": "NOT_RETRYABLE", "state": job.state.value}, + ) + redriven = ret_api.run_retention_job(db, job_id, now=_now()) + return JsonResponseWithStatus( + status_code=200, content={"job": ret_api.job_summary(redriven)} + ) + finally: + db.close() diff --git a/src/backend/routers/research/providers.py b/src/backend/routers/research/providers.py new file mode 100644 index 0000000..15e199c --- /dev/null +++ b/src/backend/routers/research/providers.py @@ -0,0 +1,449 @@ +"""Administrator-managed provider connections and researcher grants. + +Mounted under ``/api/research``. A ``provider_connection`` owns the upstream +``base_url`` and the *name* of the deployment secret (``secret_ref``). Responses +never include the secret value; the env-var name and endpoint URL are exposed to +administrators only. Researchers see only the connections explicitly granted to +them, identified by id/label with their allowed models and a readiness flag. +""" + +from __future__ import annotations + +import json +import os +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, ConfigDict, Field, field_validator +from sqlalchemy.exc import IntegrityError + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, + require_admin, +) +from backend.routers.research.access import require_researcher +from database import crud +from research.budget.pricing import parse_usd_per_million + +router = APIRouter() + + +def _parse_models(models_json: Any) -> list[str]: + try: + parsed = json.loads(models_json or "[]") + except (json.JSONDecodeError, TypeError): + return [] + if not isinstance(parsed, list): + return [] + return [str(item) for item in parsed] + + +def _secret_present(secret_ref: Optional[str]) -> bool: + """Whether the deployment environment currently holds the secret value.""" + name = (secret_ref or "").strip() + return bool(name) and bool(os.getenv(name, "").strip()) + + +def _price_payload(row: Any) -> dict[str, Any]: + return { + "input_usd_per_million": str(row.input_usd_per_million), + "output_usd_per_million": str(row.output_usd_per_million), + "cached_input_usd_per_million": ( + None + if row.cached_input_usd_per_million is None + else str(row.cached_input_usd_per_million) + ), + "updated_at": ( + row.updated_at.isoformat() if getattr(row, "updated_at", None) else None + ), + } + + +def _safe_payload( + connection: Any, + *, + admin: bool, + profile_count: Optional[int] = None, + prices: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + """Never include the secret value. Endpoint/ref name are admin-only. + + ``profile_count`` (admin only) is how many agent profiles, archived ones + included, reference the connection; any reference blocks deletion. + ``prices`` (``{model: price_row}``) drives the per-model budget prices + every caller may see: researchers size participant budgets with them. + """ + models = _parse_models(connection.models_json) + prices = prices or {} + payload: dict[str, Any] = { + "connection_id": str(connection.connection_id), + "label": connection.label, + "models": models, + "is_active": bool(connection.is_active), + # Readiness is derived, never a stored credential. + "ready": _secret_present(connection.secret_ref) and bool(connection.is_active), + # USD per million tokens per model; a metered arm whose model has no + # price fails closed (503 price_missing), so completeness is shown. + "model_prices": { + model: (_price_payload(prices[model]) if model in prices else None) + for model in models + }, + "pricing": { + "complete": all(model in prices for model in models), + "missing_models": [model for model in models if model not in prices], + }, + } + if admin: + payload["base_url"] = connection.base_url + # Name of the environment variable only — never its value. + payload["secret_ref"] = connection.secret_ref + payload["profile_count"] = int(profile_count or 0) + return payload + + +def _label_exists_error() -> HTTPException: + """The one typed duplicate-label conflict, shared by create and update.""" + return HTTPException( + status_code=409, + detail={ + "code": "CONNECTION_LABEL_EXISTS", + "field": "label", + "message": "A connection with that label already exists", + }, + ) + + +def _connection_in_use_error(profile_count: int) -> HTTPException: + return HTTPException( + status_code=409, + detail={ + "code": "CONNECTION_IN_USE", + "message": ( + f"The connection is still used by {profile_count} agent " + "profile(s), archived profiles included; point them at another " + "connection before deleting it" + ), + "profile_count": profile_count, + }, + ) + + +class ModelPricePayload(BaseModel): + """USD per million tokens for one model (decimal strings, 6 dp max).""" + + model_config = ConfigDict(extra="forbid") + + input_usd_per_million: str + output_usd_per_million: str + cached_input_usd_per_million: Optional[str] = None + + @field_validator("input_usd_per_million", "output_usd_per_million") + @classmethod + def validate_price(cls, value: str) -> str: + return str(parse_usd_per_million(value)) + + @field_validator("cached_input_usd_per_million") + @classmethod + def validate_cached_price(cls, value: Optional[str]) -> Optional[str]: + if value is None or not str(value).strip(): + return None + return str(parse_usd_per_million(value)) + + +class ProviderConnectionPayload(BaseModel): + """Admin-maintained connection; ``extra=forbid`` blocks typo'd fields.""" + + model_config = ConfigDict(extra="forbid") + + label: str = Field(..., min_length=1) + base_url: str = Field(..., min_length=1) + # Name of the deployment env var holding the key, never the key itself. + secret_ref: str = Field(..., min_length=1) + models: list[str] = Field(default_factory=list) + is_active: bool = True + # Optional per-model budget prices. Omitted (None) leaves the stored + # prices unchanged; ``{}`` clears them; a ``null`` entry deletes one. + model_prices: Optional[dict[str, Optional[ModelPricePayload]]] = None + + @field_validator("label", "base_url", "secret_ref") + @classmethod + def normalize_text(cls, value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError("value must not be blank") + return normalized + + @field_validator("base_url") + @classmethod + def validate_base_url(cls, value: str) -> str: + if not (value.startswith("http://") or value.startswith("https://")): + raise ValueError("base_url must be an http(s) endpoint") + if any(ch in value for ch in ("#",)) or "@" in value.split("//", 1)[-1]: + raise ValueError("base_url must not contain credentials or a fragment") + return value.rstrip("/") + + @field_validator("secret_ref") + @classmethod + def validate_secret_ref(cls, value: str) -> str: + """The field is an env-var *name*, never a pasted credential.""" + import re + + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,127}", value): + raise ValueError( + "secret_ref must be the NAME of an environment variable " + "(e.g. EXAMPLE_PROVIDER_KEY), not a key value" + ) + return value + + @field_validator("models") + @classmethod + def validate_models(cls, value: list[str]) -> list[str]: + cleaned = [str(item).strip() for item in value if str(item).strip()] + if not cleaned: + raise ValueError("at least one allowed model is required") + return sorted(set(cleaned)) + + +def _connection_payload_json(payload: ProviderConnectionPayload) -> str: + return json.dumps(payload.models) + + +def _price_rows(payload: ProviderConnectionPayload) -> Optional[dict[str, Optional[dict[str, Any]]]]: + """Validated ``{model: {...}|None}`` for the CRUD helper, or ``None`` (unchanged).""" + if payload.model_prices is None: + return None + if payload.model_prices == {}: + # An empty map clears every price of the connection. + return {model: None for model in payload.models} + unknown = sorted(model for model in payload.model_prices if model not in payload.models) + if unknown: + raise HTTPException( + status_code=422, + detail={ + "code": "PRICE_MODEL_UNKNOWN", + "field": "model_prices", + "message": "prices may only be set for the connection's allowed models: " + + ", ".join(unknown), + "models": unknown, + }, + ) + return { + model: ( + None + if price is None + else { + "input_usd_per_million": parse_usd_per_million(price.input_usd_per_million), + "output_usd_per_million": parse_usd_per_million(price.output_usd_per_million), + "cached_input_usd_per_million": ( + None + if price.cached_input_usd_per_million is None + else parse_usd_per_million(price.cached_input_usd_per_million) + ), + } + ) + for model, price in payload.model_prices.items() + } + + +def _prices_for(db: Any, connection_id: uuid.UUID) -> dict[str, Any]: + return {row.model: row for row in crud.list_model_prices(db, connection_id)} + + +@router.get( + "/provider-connections", + summary="List provider connections available to the caller", +) +def list_provider_connections( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Administrators see all connections; researchers see only granted ones.""" + require_researcher(current_user) + db = app.get_db_session() + try: + profile_counts: dict[Any, int] = {} + if current_user.is_admin: + connections = crud.list_provider_connections(db) + profile_counts = crud.count_profiles_by_connection(db) + else: + connections = crud.list_available_provider_connections( + db, current_user.user_id + ) + prices_by_connection = crud.list_model_prices_by_connection(db) + return JsonResponseWithStatus( + status_code=200, + content={ + "connections": [ + _safe_payload( + connection, + admin=current_user.is_admin, + profile_count=profile_counts.get(connection.connection_id, 0), + prices=prices_by_connection.get(connection.connection_id, {}), + ) + for connection in connections + ] + }, + ) + finally: + db.close() + + +@router.post( + "/provider-connections", + summary="Create a provider connection (admin)", +) +def create_provider_connection( + payload: ProviderConnectionPayload, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + if crud.get_provider_connection_by_label(db, payload.label) is not None: + raise _label_exists_error() + prices = _price_rows(payload) + try: + connection = crud.create_provider_connection( + db, + label=payload.label, + base_url=payload.base_url, + secret_ref=payload.secret_ref, + models_json=_connection_payload_json(payload), + is_active=payload.is_active, + ) + except IntegrityError as exc: + # A concurrent create won the unique label between the check above + # and the insert. + db.rollback() + raise _label_exists_error() from exc + if prices is not None: + crud.replace_model_prices( + db, connection.connection_id, prices, + allowed_models=payload.models, updated_by=current_user.email, + ) + return JsonResponseWithStatus( + status_code=201, + content={ + "connection": _safe_payload( + connection, + admin=True, + profile_count=crud.count_profiles_for_connection( + db, connection.connection_id + ), + prices=_prices_for(db, connection.connection_id), + ) + }, + ) + finally: + db.close() + + +@router.put( + "/provider-connections/{connection_id}", + summary="Update a provider connection (admin)", +) +def update_provider_connection( + connection_id: uuid.UUID, + payload: ProviderConnectionPayload, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + if crud.get_provider_connection(db, connection_id) is None: + raise HTTPException(status_code=404, detail="Connection not found") + holder = crud.get_provider_connection_by_label(db, payload.label) + if holder is not None and holder.connection_id != connection_id: + raise _label_exists_error() + prices = _price_rows(payload) + try: + connection = crud.update_provider_connection( + db, + connection_id, + label=payload.label, + base_url=payload.base_url, + secret_ref=payload.secret_ref, + models_json=_connection_payload_json(payload), + is_active=payload.is_active, + ) + except IntegrityError as exc: + # Unique-label race with a concurrent create/rename. + db.rollback() + raise _label_exists_error() from exc + if connection is None: + raise HTTPException(status_code=404, detail="Connection not found") + if prices is not None: + crud.replace_model_prices( + db, connection_id, prices, + allowed_models=payload.models, updated_by=current_user.email, + ) + else: + # Models removed from the allowlist drop their (now unusable) price. + crud.replace_model_prices( + db, connection_id, {}, allowed_models=payload.models, + updated_by=current_user.email, + ) + return JsonResponseWithStatus( + status_code=200, + content={ + "connection": _safe_payload( + connection, + admin=True, + profile_count=crud.count_profiles_for_connection( + db, connection_id + ), + prices=_prices_for(db, connection_id), + ) + }, + ) + finally: + db.close() + + +@router.delete( + "/provider-connections/{connection_id}", + summary="Delete a provider connection (admin)", +) +def delete_provider_connection( + connection_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + """Delete an unreferenced connection. + + A connection still referenced by any agent profile (archived included) is + refused with a typed 409 ``CONNECTION_IN_USE``; the ``RESTRICT`` foreign key + is the backstop for a concurrent reference, and its violation is rolled + back so the session is never left in a failed state. + """ + require_admin(current_user) + db = app.get_db_session() + try: + if crud.get_provider_connection(db, connection_id) is None: + raise HTTPException(status_code=404, detail="Connection not found") + in_use = crud.count_profiles_for_connection(db, connection_id) + if in_use: + raise _connection_in_use_error(in_use) + try: + deleted = crud.delete_provider_connection(db, connection_id) + except IntegrityError as exc: + db.rollback() + raise _connection_in_use_error( + max(1, crud.count_profiles_for_connection(db, connection_id)) + ) from exc + if not deleted: + raise HTTPException(status_code=404, detail="Connection not found") + return JsonResponseWithStatus( + status_code=200, + content={"deleted": True, "connection_id": str(connection_id)}, + ) + finally: + db.close() + + diff --git a/src/backend/routers/research/read_models.py b/src/backend/routers/research/read_models.py new file mode 100644 index 0000000..21eeac6 --- /dev/null +++ b/src/backend/routers/research/read_models.py @@ -0,0 +1,86 @@ +"""Study/profile-scoped researcher read models.""" + +from __future__ import annotations + +import uuid +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import AuthenticatedUser, get_current_user +from research.analysis.read_models import service as read_service +from research.analysis.read_models import store as read_store +from research.study.protocol import store as study_store + +operations_router = APIRouter() + + +def _authorize(db: Any, current_user: AuthenticatedUser, study_id: uuid.UUID) -> Any: + from backend.routers.research.access import require_study_owner + + study = study_store.get_study(db, study_id) + if study is None: + raise HTTPException(status_code=404, detail="Study not found") + require_study_owner(current_user, study) + return study + + +@operations_router.get("/enrollments/coverage", summary="Enrollment coverage for a study") +def enrollment_coverage( + study_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + _authorize(db, current_user, study_id) + model = read_service.build_enrollment_coverage( + read_store.list_enrollments(db, study_id), study_id=study_id + ) + return JsonResponseWithStatus(status_code=200, content=model.model_dump(mode="json")) + finally: + db.close() + + +@operations_router.get("/telemetry-coverage", summary="Telemetry coverage for a study") +def telemetry_coverage( + study_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + _authorize(db, current_user, study_id) + model = read_service.build_telemetry_coverage( + read_store.list_events(db, study_id), study_id=study_id + ) + return JsonResponseWithStatus(status_code=200, content=model.model_dump(mode="json")) + finally: + db.close() + + +@operations_router.get( + "/participants/coverage", + summary="Per-participant coverage for a study (study owner/admin only)", +) +def participant_coverage( + study_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + _authorize(db, current_user, study_id) + inputs = read_store.list_coverage_inputs(db, study_id) + model = read_service.build_participant_coverage( + inputs.enrollments, + inputs.assignments, + inputs.sessions, + inputs.events, + study_id=study_id, + ) + return JsonResponseWithStatus(status_code=200, content=model.model_dump(mode="json")) + finally: + db.close() diff --git a/src/backend/routers/research/researchers.py b/src/backend/routers/research/researchers.py new file mode 100644 index 0000000..42316d1 --- /dev/null +++ b/src/backend/routers/research/researchers.py @@ -0,0 +1,180 @@ +"""Administrator account management (P3). + +Mounted under ``/api/research/researchers``. An administrator lists every +account and flips the single ``user.can_research`` flag that lets an account own +private profiles and create studies. There is no self-service route: no +non-administrator can change ``can_research`` or ``is_admin``, and there is no +admin-promotion endpoint. + +The listing is administrator-only, so it may link an account to the studies it +is enrolled in (study name/status, enrollment status). It never includes the +study-local participant code, the enrollment id, the assigned profile or the +enrollment time, and the account's join time is reduced to its UTC date. + +This is not unlinkability. Administrators are trusted operators (they also +operate the database): study membership, the enrollment status, the join day +and the listing order (newest account first) can still single out a +participant. For example, someone who signs up just before joining has a join +day that matches the enrollment time the study analytics show, and in a small +study the order alone may match. The pseudonymisation keeps identities from +researchers, whose views never show an account. +""" + +from __future__ import annotations + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import Any, Literal, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, ConfigDict + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + require_admin, +) +from database import crud + +router = APIRouter() + +#: Upper bound so one administrator request can never materialize every account +#: ever created into a single response. +MAX_ACCOUNT_PAGE = 500 + +#: ``researcher`` = ``can_research`` and not admin; ``participant`` = neither. +AccountRoleFilter = Literal["all", "admin", "researcher", "participant"] +#: ``enrolled`` = has an ACTIVE enrollment; ``not_enrolled`` = has none. +AccountEnrollmentFilter = Literal["any", "enrolled", "not_enrolled"] + + +class ResearcherEnableRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + can_research: bool + + +def _utc_date(value: Any) -> Optional[str]: + """The UTC calendar day of ``value``; the listing needs no finer time.""" + if not isinstance(value, datetime): + return None + if value.tzinfo is not None: + value = value.astimezone(timezone.utc) + return value.date().isoformat() + + +def _user_payload(user) -> dict: + return { + "user_id": str(user.user_id), + "email": user.email, + "name": user.name, + "is_admin": bool(user.is_admin), + "can_research": bool(user.can_research), + "verified": bool(user.verified), + } + + +def _enrollment_summary(row: Any) -> dict[str, Any]: + """One enrollment of an account: study identity and statuses only. + + The enrollment id and the enrollment time are left out: study telemetry is + keyed by the id, and the study analytics list each pseudonymous participant + with its exact enrollment time, so either would link an account to its + pseudonym. The list is still ordered newest first. + """ + return { + "study_id": str(row.study_id), + "study_name": row.study_name, + "study_status": row.study_status, + "status": row.status, + } + + +def _account_payload(user, enrollments: list) -> dict[str, Any]: + payload = _user_payload(user) + payload["joined_at"] = _utc_date(getattr(user, "joined_at", None)) + payload["enrollments"] = [_enrollment_summary(row) for row in enrollments] + return payload + + +@router.get("", summary="List accounts (admin)") +def list_accounts( + q: Optional[str] = Query( + None, + max_length=200, + description="Case-insensitive substring of the account email or name.", + ), + role: AccountRoleFilter = Query( + "all", + description="admin | researcher (can_research, not admin) | " + "participant (neither) | all", + ), + enrollment: AccountEnrollmentFilter = Query( + "any", + description="enrolled (has an ACTIVE enrollment) | not_enrolled | any", + ), + study_id: Optional[uuid.UUID] = Query( + None, description="Only accounts with any enrollment in this study." + ), + limit: int = Query(100, ge=1, le=MAX_ACCOUNT_PAGE), + offset: int = Query(0, ge=0), + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + """All accounts (newest first), so an administrator can enable researchers. + + Despite the route name this is the full account list, not only the accounts + already enabled for research: the toggle in the admin UI needs both states. + ``total`` counts every account matching the filters before paging. Each row + lists the account's enrollments (newest first) with the study name and + status, loaded for the whole page in one query (no per-account queries). + """ + require_admin(current_user) + db = app.get_db_session() + try: + filters = { + "q": q, + "role": role, + "enrollment": enrollment, + "study_id": study_id, + } + total = crud.count_accounts(db, **filters) + users = crud.list_accounts(db, limit, offset=offset, **filters) + enrollments = crud.list_account_enrollments( + db, [user.user_id for user in users] + ) + return JsonResponseWithStatus( + status_code=200, + content={ + "researchers": [ + _account_payload(user, enrollments.get(user.user_id, [])) + for user in users + ], + "total": total, + "limit": limit, + "offset": offset, + }, + ) + finally: + db.close() + + +@router.put("/{user_id}", summary="Enable or disable a researcher account (admin)") +def set_researcher_enabled( + user_id: uuid.UUID, + payload: ResearcherEnableRequest, + current_user: AuthenticatedUser = Depends(require_admin), + app: App = Depends(App.get_instance), +): + require_admin(current_user) + db = app.get_db_session() + try: + user = crud.set_user_can_research(db, user_id, payload.can_research) + if user is None: + raise HTTPException(status_code=404, detail="User not found") + return JsonResponseWithStatus( + status_code=200, content={"user": _user_payload(user)} + ) + finally: + db.close() diff --git a/src/backend/routers/research/sessions.py b/src/backend/routers/research/sessions.py new file mode 100644 index 0000000..8f85e5a --- /dev/null +++ b/src/backend/routers/research/sessions.py @@ -0,0 +1,645 @@ +"""Research session lifecycle API (Issue 07). + +Mounted under ``/api/research/sessions``. Endpoints are authenticated by the +short-lived, scoped session capability issued at bootstrap +(:func:`research.runtime.bootstrap.capability.verify_capability`); the enrollment's +current revocation epoch is rechecked on every call. Handlers are thin: the +state machine lives in :mod:`research.runtime.sessions.service` and persistence in +:mod:`research.runtime.sessions.store`. + +Server time is authoritative for idle/resume/expiry decisions; a client clock is +never trusted. +""" + +from __future__ import annotations + +import logging + +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.research import bootstrap as bootstrap_router +from backend.routers.research.bootstrap import BOOTSTRAP_SIGNING_SECRET +from database.db_schemas import ResearchStudyStatus, Study as StudyRow +from research.analysis.operations import store as operations_store +from research.participants import identity as identity_store +from research.participants.enums import EnrollmentStatus +from research.runtime.bootstrap.capability import verify_capability +from research.runtime.bootstrap.models import ( + SessionCapability, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) +from research.study.protocol import store as protocol_store +from research.runtime.sessions import store as session_store +from research.runtime.sessions.enums import CloseReason, SessionReasonCode, SessionState +from research.runtime.sessions.service import ( + close, + expire_if_idle, + on_qualifying_activity, + open_session, + record_liveness, + session_policy_from_study, +) + +if TYPE_CHECKING: + from research.runtime.sessions.models import ResearchSessionV1, SessionPolicyV1 + +router = APIRouter() + + +def _budget_block(db, enrollment_id, now): + """The participant's arm-blind budget numbers, or ``None`` (never raises).""" + from research.budget import ledger as budget_ledger + + try: + return budget_ledger.participant_view(db, enrollment_id=enrollment_id, now=now) + except Exception: # noqa: BLE001 - a budget read must never break a session signal + logging.exception("[Research/sessions] budget view failed for %s", enrollment_id) + return None + +AUDIENCE = "research-runtime" +_SCOPE_WRITE = "telemetry:write" + + +def _signing_secret() -> str: + """The signing secret at request time. + + Delegates to the bootstrap router's resolver so the signer and every + verifier agree (an installed test signer wins there too); this module's + import-time copy is only a patch target of last resort. + """ + return bootstrap_router.signing_secret() or BOOTSTRAP_SIGNING_SECRET or "" +_SCOPE_HEARTBEAT = "session:heartbeat" +_SCOPE_CLOSE = "session:close" + + +class CreateSessionRequest(BaseModel): + """Create (or reuse) a research session for an active enrollment.""" + + capability: SessionCapability + enrollment_id: uuid.UUID + study_id: uuid.UUID + manifest_digest: str + # Opaque execution-context id for this project/window (never a path). + # Required: one context maps to one live session; different contexts differ. + context_id: str + environment_ref: Optional[str] = None + + +class HeartbeatRequest(BaseModel): + """Liveness heartbeat for an existing session. + + Carries no activity claim: heartbeats never update qualifying activity + (ISSUE-06). Explicit activity is reported through :class:`ActivityRequest`. + """ + + capability: SessionCapability + research_session_id: uuid.UUID + + +class ActivityRequest(BaseModel): + """One qualifying-activity report for an existing session. + + Qualifying activity is an IDE interaction / agent turn / user action that + the client observed; it is the only signal that refreshes + ``last_activity_at`` and starts a not-started session. + """ + + capability: SessionCapability + research_session_id: uuid.UUID + + +class CloseSessionRequest(BaseModel): + """Close a session with a typed reason.""" + + capability: SessionCapability + research_session_id: uuid.UUID + reason: CloseReason + + +class SessionSummaryRequest(BaseModel): + """Fetch the summary of an existing session.""" + + capability: SessionCapability + research_session_id: uuid.UUID + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _detail(code: str, message: str, **extra: Any) -> dict[str, Any]: + detail = {"code": code, "message": message} + detail.update(extra) + return detail + + +def _kill_switch_for_session(db: Any, session: ResearchSessionV1) -> Any: + """DB-backed kill-switch predicate scoped to this session's subject.""" + return operations_store.db_kill_switch_check( + db, + study_id=session.study_id, + enrollment_id=session.enrollment_id, + ) + + +def _authorize( + capability: SessionCapability, + enrollment: Any, + *, + scope: str, + now: datetime, + research_session_id: Optional[uuid.UUID] = None, + study_id: Optional[uuid.UUID] = None, +) -> None: + """Verify the capability and recheck the enrollment's revocation epoch. + + ``research_session_id`` and ``study_id`` bind the capability to the exact + session/revision being acted on, so a valid capability for another session + is rejected with a typed reason. + """ + verification = verify_capability( + capability, + _signing_secret(), + expected_audience=AUDIENCE, + expected_scope=[scope], + now=now, + current_revocation_epoch=enrollment.revocation_epoch, + expected_enrollment_id=enrollment.enrollment_id, + expected_research_session_id=research_session_id, + expected_study_id=study_id, + ) + if not verification.ok: + raise HTTPException( + status_code=403, + detail=_detail( + SessionReasonCode.CAPABILITY_INVALID.value, + verification.message or "session capability is invalid", + capability_reason=verification.reason.value, + ), + ) + if enrollment.status != EnrollmentStatus.ACTIVE: + raise HTTPException( + status_code=403, + detail=_detail( + SessionReasonCode.ENROLLMENT_NOT_ACTIVE.value, + f"enrollment is {enrollment.status.value}; bootstrap is blocked", + ), + ) + + +def _load_enrollment(db: Any, enrollment_id: uuid.UUID): + row = identity_store.get_enrollment(db, enrollment_id) + if row is None: + raise HTTPException(status_code=404, detail="Enrollment not found") + study = db.get(StudyRow, row.study_id) + if getattr(study, "research_status", None) == ResearchStudyStatus.STUDY_STOPPED.value: + raise HTTPException( + status_code=403, + detail=_detail( + "STUDY_STOPPED", + "the study has been stopped and cannot accept session activity", + ), + ) + return identity_store.row_to_enrollment(row) + + +def _load_session(db: Any, research_session_id: uuid.UUID) -> ResearchSessionV1: + row = session_store.get_session(db, research_session_id) + if row is None: + raise HTTPException(status_code=404, detail="Research session not found") + return session_store.row_to_session(row) + + +def _load_policy(db: Any, session: ResearchSessionV1) -> Optional[SessionPolicyV1]: + study = db.get(StudyRow, session.study_id) + return session_policy_from_study(study) + + +def _persist_result(db: Any, result: Any) -> None: + """Persist a session result: append the transition (if any) and always + update the session row. + + Updating unconditionally is what makes an *ordinary* heartbeat durable: a + heartbeat that only refreshes ``last_activity_at`` has no transition, so + gating the row update on ``transition is not None`` would silently drop it. + """ + if result.transition is not None: + session_store.insert_transition(db, result.transition) + session_store.update_session(db, result.session) + + +def _next_actions( + session: ResearchSessionV1, policy: SessionPolicyV1, now: datetime +) -> list[str]: + if session.state == SessionState.NOT_STARTED: + return ["report_activity"] + if session.state == SessionState.RUNNING: + return ["heartbeat"] + if session.state == SessionState.OFFLINE: + return ["recover"] + if session.state == SessionState.SUSPENDED: + last = session.last_activity_at + within = ( + last is not None + and (now - last).total_seconds() <= policy.resume_grace_seconds + ) + return ["resume_within_grace"] if within else ["start_new_session"] + return [] + + +@router.post("/", summary="Create (or reuse) a research session") +def create_research_session( + payload: CreateSessionRequest, + app: App = Depends(App.get_instance), +): + """Open a session for an active enrollment, or return the active one.""" + now = _now() + db = app.get_db_session() + try: + enrollment = _load_enrollment(db, payload.enrollment_id) + _authorize( + payload.capability, + enrollment, + scope=_SCOPE_WRITE, + now=now, + study_id=payload.study_id, + ) + if payload.study_id != enrollment.study_id: + raise HTTPException( + status_code=409, + detail=_detail( + "STUDY_MISMATCH", + "enrollment is not bound to the requested study", + ), + ) + study = db.get(StudyRow, enrollment.study_id) + policy = session_policy_from_study(study) + if policy is None: + raise HTTPException( + status_code=409, + detail=_detail( + SessionReasonCode.POLICY_MISSING.value, + "study does not declare idle/resume session policy", + ), + ) + # The study window is authoritative, not only the terminal status: an + # ended (past ends_at) or not-yet-started study must not open sessions. + if study is not None and not protocol_store.research_study_is_open(study, now): + raise HTTPException( + status_code=409, + detail=_detail( + SessionReasonCode.STUDY_NOT_OPEN.value, + "the study is not currently open for execution", + ), + ) + + # An idle-expired context session is ended (IDLE_TIMEOUT) and a fresh + # one opened below, instead of handing back a session the next + # heartbeat would terminate (review D-01/C-02). + kill_switch_check = operations_store.db_kill_switch_check( + db, + study_id=study.study_id, + enrollment_id=enrollment.enrollment_id, + ) + existing_row = session_store.resolve_open_context_session( + db, + enrollment.enrollment_id, + payload.context_id, + study=study, + now=now, + kill_switch_check=kill_switch_check, + ) + if existing_row is not None: + existing = session_store.row_to_session(existing_row) + return JsonResponseWithStatus( + status_code=200, + content={ + "created": False, + "session": session_store.session_summary(existing_row), + "next_actions": _next_actions(existing, policy, now), + "heartbeat_seconds": policy.heartbeat_seconds, + "budget": _budget_block(db, enrollment.enrollment_id, now), + }, + ) + + session = open_session( + enrollment, + study, + manifest_digest=payload.manifest_digest, + environment_ref=payload.environment_ref, + context_id=payload.context_id, + now=now, + ) + # An engaged kill switch blocks new funded session creation. + if kill_switch_check(): + raise HTTPException( + status_code=403, + detail=_detail( + "KILL_SWITCH_ENGAGED", + "an operator kill switch is engaged for this study", + ), + ) + row = session_store.create_session(db, session) + return JsonResponseWithStatus( + status_code=201, + content={ + "created": True, + "session": session_store.session_summary(row), + "next_actions": _next_actions(session, policy, now), + "heartbeat_seconds": policy.heartbeat_seconds, + "budget": _budget_block(db, enrollment.enrollment_id, now), + }, + ) + finally: + db.close() + + +@router.post("/heartbeat", summary="Heartbeat / report session activity") +def heartbeat( + payload: HeartbeatRequest, + app: App = Depends(App.get_instance), +): + """Update last-activity (or start the session) and return next actions.""" + now = _now() + db = app.get_db_session() + try: + session = _load_session(db, payload.research_session_id) + enrollment = _load_enrollment(db, session.enrollment_id) + _authorize( + payload.capability, + enrollment, + scope=_SCOPE_HEARTBEAT, + now=now, + research_session_id=session.research_session_id, + study_id=session.study_id, + ) + + if session.state.is_terminal: + raise HTTPException( + status_code=409, + detail=_detail( + SessionReasonCode.SESSION_TERMINAL.value, + f"session is {session.state.value}", + ), + ) + + policy = _load_policy(db, session) + if policy is None: + raise HTTPException( + status_code=409, + detail=_detail( + SessionReasonCode.POLICY_MISSING.value, + "study does not declare idle/resume session policy", + ), + ) + + kill_switch_check = _kill_switch_for_session(db, session) + idle_result = expire_if_idle( + session, now, policy=policy, kill_switch_check=kill_switch_check + ) + if not idle_result.accepted: + raise HTTPException( + status_code=409, + detail=_detail( + idle_result.reason.value, + ( + idle_result.issue.message + if idle_result.issue is not None + else "heartbeat rejected" + ), + ), + ) + if idle_result.transition is not None: + _persist_result(db, idle_result) + row = session_store.get_session(db, session.research_session_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "session": session_store.session_summary(row), + "next_actions": [], + "heartbeat_seconds": policy.heartbeat_seconds, + "budget": _budget_block(db, session.enrollment_id, now), + }, + ) + + liveness = record_liveness(idle_result.session, now) + if not liveness.accepted: + raise HTTPException( + status_code=409, + detail=_detail( + liveness.reason.value, + ( + liveness.issue.message + if liveness.issue is not None + else "heartbeat rejected" + ), + ), + ) + _persist_result(db, liveness) + + row = session_store.get_session(db, session.research_session_id) + updated = session_store.row_to_session(row) + return JsonResponseWithStatus( + status_code=200, + content={ + "session": session_store.session_summary(row), + "next_actions": _next_actions(updated, policy, now), + "heartbeat_seconds": policy.heartbeat_seconds, + "budget": _budget_block(db, session.enrollment_id, now), + }, + ) + finally: + db.close() + + +@router.post("/activity", summary="Report qualifying session activity") +def report_activity( + payload: ActivityRequest, + app: App = Depends(App.get_instance), +): + """Refresh qualifying activity (and start a not-started session). + + This is the only client-signalled path that updates ``last_activity_at``; + heartbeats are liveness only (ISSUE-06). + """ + now = _now() + db = app.get_db_session() + try: + session = _load_session(db, payload.research_session_id) + enrollment = _load_enrollment(db, session.enrollment_id) + _authorize( + payload.capability, + enrollment, + scope=_SCOPE_HEARTBEAT, + now=now, + research_session_id=session.research_session_id, + study_id=session.study_id, + ) + + if session.state.is_terminal: + raise HTTPException( + status_code=409, + detail=_detail( + SessionReasonCode.SESSION_TERMINAL.value, + f"session is {session.state.value}", + ), + ) + + policy = _load_policy(db, session) + if policy is None: + raise HTTPException( + status_code=409, + detail=_detail( + SessionReasonCode.POLICY_MISSING.value, + "study does not declare idle/resume session policy", + ), + ) + + kill_switch_check = _kill_switch_for_session(db, session) + idle_result = expire_if_idle( + session, now, policy=policy, kill_switch_check=kill_switch_check + ) + if not idle_result.accepted: + raise HTTPException( + status_code=409, + detail=_detail( + idle_result.reason.value, + ( + idle_result.issue.message + if idle_result.issue is not None + else "activity rejected" + ), + ), + ) + if idle_result.transition is not None: + _persist_result(db, idle_result) + row = session_store.get_session(db, session.research_session_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "session": session_store.session_summary(row), + "next_actions": [], + "heartbeat_seconds": policy.heartbeat_seconds, + "budget": _budget_block(db, session.enrollment_id, now), + }, + ) + + activity = on_qualifying_activity( + idle_result.session, + now, + kill_switch_check=kill_switch_check, + idle_timeout_seconds=policy.idle_timeout_seconds, + ) + if not activity.accepted: + raise HTTPException( + status_code=409, + detail=_detail( + activity.reason.value, + ( + activity.issue.message + if activity.issue is not None + else "activity rejected" + ), + ), + ) + _persist_result(db, activity) + + row = session_store.get_session(db, session.research_session_id) + return JsonResponseWithStatus( + status_code=200, + content={ + "session": session_store.session_summary(row), + "next_actions": _next_actions(activity.session, policy, now), + "heartbeat_seconds": policy.heartbeat_seconds, + "budget": _budget_block(db, session.enrollment_id, now), + }, + ) + finally: + db.close() + + +@router.post("/close", summary="Close a research session") +def close_research_session( + payload: CloseSessionRequest, + app: App = Depends(App.get_instance), +): + """Apply a typed terminal close reason.""" + now = _now() + db = app.get_db_session() + try: + session = _load_session(db, payload.research_session_id) + enrollment = _load_enrollment(db, session.enrollment_id) + _authorize( + payload.capability, + enrollment, + scope=_SCOPE_CLOSE, + now=now, + research_session_id=session.research_session_id, + study_id=session.study_id, + ) + + if session.state.is_terminal: + raise HTTPException( + status_code=409, + detail=_detail( + SessionReasonCode.SESSION_TERMINAL.value, + f"session is {session.state.value}", + ), + ) + + result = close( + session, + payload.reason, + now, + kill_switch_check=_kill_switch_for_session(db, session), + ) + if not result.accepted: + raise HTTPException( + status_code=409, + detail=_detail( + result.reason.value, + result.issue.message if result.issue else "close rejected", + ), + ) + _persist_result(db, result) + row = session_store.get_session(db, session.research_session_id) + return JsonResponseWithStatus( + status_code=200, + content={"session": session_store.session_summary(row), "next_actions": []}, + ) + finally: + db.close() + + +@router.post("/summary", summary="Get a research session summary") +def get_research_session( + payload: SessionSummaryRequest, + app: App = Depends(App.get_instance), +): + """Return the summary of a session, authorized by the session capability.""" + now = _now() + db = app.get_db_session() + try: + session = _load_session(db, payload.research_session_id) + enrollment = _load_enrollment(db, session.enrollment_id) + _authorize( + payload.capability, + enrollment, + scope=_SCOPE_WRITE, + now=now, + research_session_id=session.research_session_id, + study_id=session.study_id, + ) + row = session_store.get_session(db, session.research_session_id) + return JsonResponseWithStatus( + status_code=200, + content={"session": session_store.session_summary(row)}, + ) + finally: + db.close() diff --git a/src/backend/routers/research/studies.py b/src/backend/routers/research/studies.py new file mode 100644 index 0000000..5e34d08 --- /dev/null +++ b/src/backend/routers/research/studies.py @@ -0,0 +1,723 @@ +"""Lifecycle API for research studies. + +A research study has one configuration authority on ``public.study``. The old +revision/draft/publication API was removed; study configuration is created once, +metadata can be edited before consent, and stop is terminal and non-destructive. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import Any, Optional, cast + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator +from sqlalchemy import select + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import AuthenticatedUser, get_current_user +from backend.routers.research.access import require_researcher, require_study_owner +from backend.routers.research.join import study_consent_view +from database.db_schemas import AgentProfile +from database.db_schemas import Study as StudyRow +from research.analysis.operations import store as operations_store +from research.budget import study_policy +from research.budget.pricing import parse_usd_amount, usd_to_micro +from research.runtime.assignment.hashing import ( + assignment_strategy, + manual_override_enabled, + new_study_assignment_policy, +) +from research.runtime.sessions.models import SessionPolicyV1 +from research.study import consent +from research.study.agents.enums import METERED_FRAMEWORKS +from research.study.lifecycle import ( + AssignmentOverrideError, + CloneNotAllowedError, + ResearchStudyNotFoundError, + allocate_join_code, + clone_stopped_research_study, + reassign_enrollment, + revoke_research_enrollment, + stop_research_study, + update_research_metadata, +) +from research.study.protocol import store + +router = APIRouter() + + +class StudyCreateRequest(BaseModel): + """Create one research study configuration.""" + + name: str = Field(..., min_length=1) + description: Optional[str] = None + starts_at: Optional[datetime] = None + ends_at: Optional[datetime] = None + telemetry_policy: dict[str, Any] = Field(default_factory=dict) + session_policy: dict[str, Any] = Field(default_factory=dict) + # Required and non-empty: a research study is created with its fixed profile + # selection. The clone route re-validates any selection it carries through + # the same freeze; a clone body may omit profiles and stay profile-less. + profile_ids: list[uuid.UUID] = Field(..., min_length=1) + # Participant budgets (shared provider key). Required (> 0) when a selected + # profile runs Goose or the built-in agent; ignored for Codex-only studies. + default_budget_usd: Optional[str] = None + budget_warning_fraction: float = Field(default=0.8, gt=0, le=1) + # Arms are always drawn by salted hash; this only lets the owner change a + # participant's arm by hand before first use (disclosed in the consent text). + allow_manual_assignment: bool = False + # A custom consent form (document + tick-box statements); omitted, the stock + # notice applies. Frozen with the configuration like the policies. + consent: Optional[dict[str, Any]] = None + + @field_validator("name") + @classmethod + def validate_name(cls, value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError("name must not be blank") + return normalized + + +class StudyCloneRequest(BaseModel): + """Optional profile selection completing a stopped-study clone. + + Supplying ``profile_ids`` runs the same validation/freeze as create, so the + clone is joinable. Omitting the body keeps the clone profile-less; such a + clone is explicitly unusable and join refuses it until a profile-bearing + clone is requested (ISSUE-12). + """ + + profile_ids: Optional[list[uuid.UUID]] = Field(default=None, min_length=1) + # Override for the clone's default participant budget (else copied). + default_budget_usd: Optional[str] = None + + +class StudyMetadataUpdateRequest(BaseModel): + """Metadata fields allowed before the first consent.""" + + name: Optional[str] = None + description: Optional[str] = None + starts_at: Optional[datetime] = None + ends_at: Optional[datetime] = None + + +class StopStudyRequest(BaseModel): + actor: Optional[str] = None + + +class RevokeEnrollmentRequest(BaseModel): + actor: Optional[str] = None + + +class AssignmentOverrideRequest(BaseModel): + """The arm (one of the study's selected profiles) to give the participant.""" + + model_config = ConfigDict(extra="forbid") + + profile_id: uuid.UUID + + +def _load_study(db: Any, study_id: uuid.UUID): + study = store.get_study(db, study_id) + if study is None: + raise HTTPException(status_code=404, detail="Study not found") + return study + + +def _authorize_study( + db: Any, + current_user: AuthenticatedUser, + study_id: uuid.UUID, +): + study = _load_study(db, study_id) + require_study_owner(current_user, study) + return study + + +def _validated_session_policy(session_policy: dict[str, Any]) -> dict[str, Any]: + """Validate and freeze the study's complete typed session policy. + + A study whose policy the session endpoints would reject must never be + stored: creation fails with a typed 4xx instead of producing a study that + can be joined but can never open an authoritative session (ISSUE-05). + """ + try: + parsed = SessionPolicyV1.model_validate(session_policy) + except ValidationError as error: + first = error.errors()[0] + location = ".".join(str(part) for part in first.get("loc", ())) + message = first.get("msg", "invalid session policy") + raise HTTPException( + status_code=422, + detail={ + "code": "SESSION_POLICY_INVALID", + "message": f"{location}: {message}" if location else message, + }, + ) from error + if parsed.heartbeat_seconds is not None and parsed.idle_timeout_seconds <= parsed.heartbeat_seconds: + # A window shows it is alive once per heartbeat: an idle timeout no + # longer than that would end and rotate its session on every tick. + raise HTTPException( + status_code=422, + detail={ + "code": "SESSION_POLICY_INVALID", + "message": "idle_timeout_seconds must be longer than heartbeat_seconds", + }, + ) + return parsed.model_dump(mode="json") + + +def _validated_telemetry_policy(telemetry_policy: dict[str, Any]) -> dict[str, Any]: + """Reject a malformed telemetry policy at creation (ISSUE-01/ISSUE-05). + + Content capture is only ever declared, never inferred: ``content_capture`` + must be a boolean and the field-class allowlist must contain only the + known policy vocabulary. + """ + # The study-facing policy vocabulary is ``TelemetryFieldClass`` + # (STRUCTURAL / METRICS / DIAGNOSTICS / CONTENT). The runtime privacy + # vocabulary (SYSTEM / BEHAVIORAL / CODE_METADATA / CONTENT) is derived from + # it by the privacy engine, so it must not be what a researcher authors. + from research.study.protocol.enums import TelemetryFieldClass + from research.telemetry.enums import FieldClass + + if not isinstance(telemetry_policy, dict): + raise HTTPException( + status_code=422, + detail={ + "code": "TELEMETRY_POLICY_INVALID", + "message": "telemetry_policy must be an object", + }, + ) + content_capture = telemetry_policy.get("content_capture") + if content_capture is not None and not isinstance(content_capture, bool): + raise HTTPException( + status_code=422, + detail={ + "code": "TELEMETRY_POLICY_INVALID", + "message": "content_capture must be a boolean", + }, + ) + allowed = telemetry_policy.get("allowed_field_classes") + if allowed is not None: + if not isinstance(allowed, list) or not all( + isinstance(item, str) for item in allowed + ): + raise HTTPException( + status_code=422, + detail={ + "code": "TELEMETRY_POLICY_INVALID", + "message": ( + "allowed_field_classes must be a list of field-class names " + f"({', '.join(sorted({c.value for c in TelemetryFieldClass}))})" + ), + }, + ) + # Accept both vocabularies: the study-facing authoring names + # (STRUCTURAL/METRICS/DIAGNOSTICS/CONTENT) and the runtime privacy names + # (SYSTEM/BEHAVIORAL/CODE_METADATA/CONTENT) used by older callers. + if FieldClass.SECRET.value in allowed: + # SECRET is never persisted; declaring it could only be a mistake. + raise HTTPException( + status_code=422, + detail={ + "code": "TELEMETRY_POLICY_INVALID", + "message": "SECRET is never collected and cannot be allowed", + }, + ) + known = {field_class.value for field_class in TelemetryFieldClass} | { + field_class.value for field_class in FieldClass + } + unknown = sorted(set(allowed) - known) + if unknown: + raise HTTPException( + status_code=422, + detail={ + "code": "TELEMETRY_POLICY_INVALID", + "message": f"unknown field classes: {', '.join(unknown)}", + }, + ) + return telemetry_policy + + +def _budget_error(code: str, message: str, **extra: Any) -> HTTPException: + return HTTPException( + status_code=422, + detail={"code": code, "field": "default_budget_usd", "message": message, **extra}, + ) + + +def _metered_profile_rows(db: Any, profile_ids: list[uuid.UUID]) -> list[AgentProfile]: + if not profile_ids: + return [] + rows = db.execute( + select(AgentProfile).where(AgentProfile.profile_id.in_(list(profile_ids))) + ).scalars().all() + return [ + row for row in rows + if str(row.framework_version or "").strip().lower() in METERED_FRAMEWORKS + ] + + +def _resolve_default_budget( + db: Any, profile_ids: list[uuid.UUID], default_budget_usd: Optional[str] +) -> int: + """The study's default participant budget in micro-USD, validated. + + Metered arms (Goose, built-in agent) need a positive budget and a price for + their frozen model, otherwise every participant call would fail closed on + day one. Codex-only studies keep whatever was given (0 when omitted). + """ + metered = _metered_profile_rows(db, profile_ids) + if default_budget_usd is not None and str(default_budget_usd).strip(): + try: + amount = parse_usd_amount(default_budget_usd) + except ValueError as error: + raise _budget_error("BUDGET_INVALID", str(error)) from error + else: + amount = None + if not metered: + return usd_to_micro(amount) if amount is not None else 0 + if amount is None: + raise _budget_error( + "BUDGET_REQUIRED", + "a default budget per participant is required when a selected profile " + "runs Goose or the built-in agent", + ) + if amount <= 0: + raise _budget_error( + "BUDGET_INVALID", + "the default budget per participant must be greater than zero for " + "Goose/built-in arms", + ) + missing = study_policy.missing_prices( + db, + [ + { + "profile_id": str(row.profile_id), + "name": row.name, + "model": row.model, + "connection_id": str(row.connection_id) if row.connection_id else None, + } + for row in metered + ], + ) + if missing: + raise _budget_error( + "BUDGET_PRICE_MISSING", + "these profiles' models have no budget price on their provider connection; " + "ask an administrator to price them first: " + + ", ".join(f"{item['name']} ({item['model']})" for item in missing), + missing=missing, + ) + return usd_to_micro(amount) + + +def _consent_view_payload(config: Any) -> dict[str, Any]: + view = study_consent_view(config) + return { + **view, + "custom": view["document"] is not None, + "digest": consent.view_digest(view), + } + + +def _study_payload(study: Any, db: Any) -> dict[str, Any]: + def iso(value: Any) -> Any: + return value.isoformat() if isinstance(value, datetime) else value + + metrics = store.get_study_read_metrics(db, study.study_id) + stopped = getattr(study, "research_status", None) == "STUDY_STOPPED" + budget_row = db.get(StudyRow, study.study_id) + budget_policy = study_policy.budget_policy_payload( + budget_row if budget_row is not None else study, + study_policy.metered_selections(db, study.study_id), + ) + # A ``StudyView`` carries no config: the frozen policies live on the row. + config = ( + getattr(budget_row, "research_config_json", None) + or getattr(study, "research_config_json", None) + or {} + ) + switch = operations_store.latest_study_kill_switch(db, study.study_id) + switch_status = None + if switch is not None: + switch_status = "ENGAGED" if switch.is_engaged(datetime.now(timezone.utc)) else "RELEASED" + return { + "study_id": str(study.study_id), + "name": study.name, + "description": study.description, + "owner": getattr(study, "owner", None), + "created_by": str(study.created_by) if study.created_by else None, + "is_research": bool(getattr(study, "is_research", False)), + "is_active": bool(getattr(study, "is_active", False)), + "research_status": getattr(study, "research_status", None), + "research_config_digest": getattr(study, "research_config_digest", None), + "join_code": getattr(study, "join_code", None), + "consent_locked_at": iso(getattr(study, "consent_locked_at", None)), + "stopped_at": iso(getattr(study, "stopped_at", None)), + "stopped_by": getattr(study, "stopped_by", None), + "profile_selections": list(getattr(study, "profile_selections", []) or []), + # The frozen policies are echoed so the UI can display/edit the actual + # authority; metadata PATCH never changes them. + "telemetry_policy": config.get("telemetry_policy") or {}, + "session_policy": config.get("session_policy") or {}, + "assignment_policy": { + "strategy": assignment_strategy(config), + "manual_override": manual_override_enabled(config), + }, + # Exactly what participants review and accept (document, platform + # notice, statements) and its version digest. + "consent": _consent_view_payload(config), + "starts_at": iso(getattr(study, "starts_at", None)), + "ends_at": iso(getattr(study, "ends_at", None)), + "created_at": iso(getattr(study, "created_at", None)), + "enrollment_count": metrics.enrollment_count, + "active_enrollment_count": metrics.active_enrollment_count, + "assignment_count": metrics.assignment_count, + "active_assignment_count": metrics.active_assignment_count, + "active_session_count": metrics.active_session_count, + "collection_status": "STOPPED" if stopped else ( + "ACTIVE" if metrics.active_session_count else "IDLE" + ), + "lifecycle_capabilities": { + "metadata_editable": not stopped and getattr(study, "consent_locked_at", None) is None, + "stoppable": not stopped, + "cloneable": stopped, + "joinable": not stopped and getattr(study, "research_status", None) in {"DRAFT", "ACTIVE"}, + # Budgets stay editable after the consent lock; only a stop ends it. + "budget_editable": not stopped, + }, + "budget_policy": budget_policy, + "kill_switch": ( + { + "switch_id": str(switch.switch_id), + "status": "ENGAGED" if stopped else switch_status, + "reason": switch.reason, + } + if switch is not None + else None + ), + } + + +@router.post("", summary="Create a research study") +def create_study( + payload: StudyCreateRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_researcher(current_user) + db = app.get_db_session() + try: + session_policy = _validated_session_policy(payload.session_policy) + telemetry_policy = _validated_telemetry_policy(payload.telemetry_policy) + default_budget = _resolve_default_budget( + db, list(payload.profile_ids), payload.default_budget_usd + ) + research_config: dict[str, Any] = { + "telemetry_policy": telemetry_policy, + "session_policy": session_policy, + "profile_ids": [str(profile_id) for profile_id in payload.profile_ids], + "assignment": new_study_assignment_policy( + manual_override=payload.allow_manual_assignment + ), + } + if payload.consent is not None: + try: + research_config["consent"] = consent.validate_consent_config(payload.consent) + except consent.ConsentError as error: + raise HTTPException(status_code=error.status_code, detail=error.detail()) from error + study = store.create_study( + db, + study_id=uuid.uuid4(), + name=payload.name, + description=payload.description, + created_by=current_user.user_id, + starts_at=payload.starts_at, + ends_at=payload.ends_at, + is_research=True, + research_config_json=research_config, + join_code=allocate_join_code(db), + profile_ids=payload.profile_ids, + allow_shared_profiles=current_user.is_admin, + inference_budget_default_micro_usd=default_budget, + inference_budget_warning_fraction=payload.budget_warning_fraction, + inference_budget_updated_by=current_user.email, + ) + return JsonResponseWithStatus( + status_code=201, + content=cast(Any, {"study": _study_payload(study, db)}), + ) + except PermissionError as error: + db.rollback() + raise HTTPException( + status_code=403, + detail={"code": "PROFILE_NOT_ALLOWED", "message": str(error)}, + ) from error + except ValueError as error: + db.rollback() + message = str(error) + code = getattr(error, "code", None) or ( + message.split(":", 1)[0] if ":" in message else "PROFILE_NOT_ALLOWED" + ) + raise HTTPException( + status_code=422, + detail={"code": code, "message": message}, + ) from error + finally: + db.close() + + +@router.get("", summary="List the caller's research studies") +def list_studies( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_researcher(current_user) + db = app.get_db_session() + try: + owner_id = None if current_user.is_admin else current_user.user_id + studies = store.list_studies(db, owner_id) + return JsonResponseWithStatus( + status_code=200, + content=cast(Any, {"studies": [_study_payload(study, db) for study in studies]}), + ) + finally: + db.close() + + +@router.get("/{study_id}", summary="Get one research study") +def get_study( + study_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_researcher(current_user) + db = app.get_db_session() + try: + study = _authorize_study(db, current_user, study_id) + return JsonResponseWithStatus( + status_code=200, + content=cast(Any, {"study": _study_payload(study, db)}), + ) + finally: + db.close() + + +@router.patch("/{study_id}/metadata", summary="Update unlocked study metadata") +def update_study_metadata( + study_id: uuid.UUID, + payload: StudyMetadataUpdateRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_researcher(current_user) + if all(value is None for value in (payload.name, payload.description, payload.starts_at, payload.ends_at)): + raise HTTPException(status_code=422, detail="at least one metadata field is required") + db = app.get_db_session() + try: + _authorize_study(db, current_user, study_id) + try: + updated = update_research_metadata( + db, + study_id, + name=payload.name, + description=payload.description, + starts_at=payload.starts_at, + ends_at=payload.ends_at, + ) + except PermissionError as error: + raise HTTPException( + status_code=409, + detail={"code": "STUDY_METADATA_LOCKED", "message": str(error)}, + ) from error + except ValueError as error: + raise HTTPException(status_code=404, detail=str(error)) from error + return JsonResponseWithStatus( + status_code=200, + content=cast(Any, {"study": _study_payload(updated, db)}), + ) + finally: + db.close() + + +@router.post("/{study_id}/stop", summary="Stop a research study permanently") +def stop_study( + study_id: uuid.UUID, + payload: StopStudyRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_researcher(current_user) + db = app.get_db_session() + try: + _authorize_study(db, current_user, study_id) + summary = stop_research_study(db, study_id, actor=payload.actor or current_user.email) + updated = store.get_study(db, study_id) + return JsonResponseWithStatus( + status_code=200, + content=cast(Any, { + "study": _study_payload(updated, db) if updated is not None else None, + "stopped": True, + "enrollment_count": summary.enrollment_count, + "assignment_count": summary.assignment_count, + "session_count": summary.session_count, + }), + ) + finally: + db.close() + + +@router.post("/{study_id}/clone", summary="Clone a stopped research study") +def clone_study( + study_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), + payload: Optional[StudyCloneRequest] = None, +): + """Clone a stopped study, optionally completing it with profile selections. + + No body keeps the historical profile-less draft. A body carrying + ``profile_ids`` freezes those profiles with the create-time invariants and + inserts the selection rows atomically, so the clone is joinable (ISSUE-12). + """ + require_researcher(current_user) + db = app.get_db_session() + try: + _authorize_study(db, current_user, study_id) + budget_override = None + if payload is not None and payload.default_budget_usd is not None: + budget_override = _resolve_default_budget( + db, list(payload.profile_ids or []), payload.default_budget_usd + ) + elif payload is not None and payload.profile_ids: + # A clone that selects metered profiles must not silently inherit an + # unusable (zero) default from the source study. + source_row = db.get(StudyRow, study_id) + inherited = int(getattr(source_row, "inference_budget_default_micro_usd", 0) or 0) + _resolve_default_budget( + db, list(payload.profile_ids), + study_policy.micro_to_usd_string(inherited) if inherited > 0 else None, + ) + try: + clone = clone_stopped_research_study( + db, + study_id, + actor=current_user.email, + profile_ids=payload.profile_ids if payload is not None else None, + allow_shared_profiles=current_user.is_admin, + inference_budget_default_micro_usd=budget_override, + ) + except CloneNotAllowedError as error: + raise HTTPException(status_code=409, detail=str(error)) from error + except PermissionError as error: + raise HTTPException( + status_code=403, + detail={"code": "PROFILE_NOT_ALLOWED", "message": str(error)}, + ) from error + except ResearchStudyNotFoundError as error: + raise HTTPException(status_code=404, detail=str(error)) from error + except ValueError as error: + message = str(error) + code = getattr(error, "code", None) or ( + message.split(":", 1)[0] if ":" in message else "PROFILE_NOT_ALLOWED" + ) + raise HTTPException( + status_code=422, + detail={"code": code, "message": message}, + ) from error + # Re-read through the store view so the response carries the frozen + # profile selections this clone just inserted, not the bare ORM row. + clone_view = store.get_study(db, clone.study_id) or clone + return JsonResponseWithStatus( + status_code=201, + content=cast(Any, {"study": _study_payload(clone_view, db)}), + ) + finally: + db.close() + + +@router.post( + "/{study_id}/enrollments/{enrollment_id}/revoke", + summary="Revoke one participant enrollment", +) +def revoke_enrollment( + study_id: uuid.UUID, + enrollment_id: uuid.UUID, + payload: RevokeEnrollmentRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_researcher(current_user) + db = app.get_db_session() + try: + _authorize_study(db, current_user, study_id) + try: + summary = revoke_research_enrollment(db, study_id, enrollment_id) + except ValueError as error: + raise HTTPException(status_code=404, detail=str(error)) from error + return JsonResponseWithStatus( + status_code=200, + content=cast(Any, { + "revoked": True, + "enrollment_id": str(summary.enrollment_id), + "session_count": summary.session_count, + }), + ) + finally: + db.close() + + +@router.put( + "/{study_id}/enrollments/{enrollment_id}/assignment", + summary="Set a participant's arm by hand before first use (opt-in per study)", +) +def override_enrollment_assignment( + study_id: uuid.UUID, + enrollment_id: uuid.UUID, + payload: AssignmentOverrideRequest, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + require_researcher(current_user) + db = app.get_db_session() + try: + _authorize_study(db, current_user, study_id) + try: + summary = reassign_enrollment( + db, + study_id, + enrollment_id, + payload.profile_id, + actor=current_user.email, + ) + except AssignmentOverrideError as error: + db.rollback() + raise HTTPException( + status_code=error.status_code, + detail={"code": error.code, "message": str(error)}, + ) from error + except ResearchStudyNotFoundError as error: + db.rollback() + raise HTTPException(status_code=404, detail="Study not found") from error + return JsonResponseWithStatus( + status_code=200, + content=cast(Any, { + "enrollment_id": str(summary.enrollment_id), + "changed": summary.changed, + "assignment": { + "profile_id": str(summary.to_profile_id), + "strategy": summary.strategy, + "assigned_at": ( + summary.assigned_at.isoformat() if summary.assigned_at else None + ), + }, + }), + ) + finally: + db.close() diff --git a/src/backend/routers/research/study_analytics.py b/src/backend/routers/research/study_analytics.py new file mode 100644 index 0000000..7d6e613 --- /dev/null +++ b/src/backend/routers/research/study_analytics.py @@ -0,0 +1,245 @@ +"""Study analytics read models (study owner or administrator only). + +Mounted under ``/api/research/studies``: + +* ``GET /{study_id}/analytics/participants`` - participants table: each + enrollment's frozen arm, session and activity summary and health. +* ``GET /{study_id}/analytics/participants/{enrollment_id}`` - one + participant's telemetry dashboard (metric set, daily activity, tools, turns, + sessions, metadata-only timeline). +* ``GET /{study_id}/analytics/summary`` - study totals and the arm comparison on + participant-level values; optional inclusive UTC ``start``/``end`` dates. + +Everything is read-only, study-scoped and metadata-only: no content fields and +no login identity are ever read or returned. +""" + +from __future__ import annotations + +import re +import uuid # noqa: TC003 - FastAPI resolves the path-parameter annotations at runtime +from datetime import date, datetime, timezone +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import AuthenticatedUser, get_current_user +from database.db_schemas import ResearchStudyStatus +from database.db_schemas import Study as StudyRow +from research.analysis.study_analytics import metrics as analytics +from research.analysis.study_analytics import store as analytics_store +from research.analysis.study_analytics.models import DateWindow +from research.budget import ledger as budget_ledger +from research.runtime.assignment.hashing import manual_override_enabled +from research.study.lifecycle import enrollments_with_activity +from research.study.protocol import store as study_store + +router = APIRouter() + +_DAY_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +def _authorize(db: Any, current_user: AuthenticatedUser, study_id: uuid.UUID) -> Any: + """404 for an unknown study, 403 ``FORBIDDEN_STUDY`` for a non-owner. + + Ownership (owner or administrator) is the whole gate: a participant is never + an owner, so ``require_researcher`` would only change the error code the + authorization matrix pins.""" + from backend.routers.research.access import require_study_owner + + study = study_store.get_study(db, study_id) + if study is None: + raise HTTPException(status_code=404, detail="Study not found") + require_study_owner(current_user, study) + return study + + +def _enrollment_not_found() -> HTTPException: + return HTTPException( + status_code=404, + detail={ + "code": "ENROLLMENT_NOT_FOUND", + "message": "no such enrollment in this study", + }, + ) + + +def _parse_day(value: Optional[str], name: str) -> Optional[date]: + if value is None or value == "": + return None + if not _DAY_PATTERN.fullmatch(value): + raise HTTPException( + status_code=422, + detail={ + "code": "INVALID_DATE", + "message": f"{name} must be a YYYY-MM-DD date", + }, + ) + try: + return date.fromisoformat(value) + except ValueError as error: + raise HTTPException( + status_code=422, + detail={ + "code": "INVALID_DATE", + "message": f"{name} is not a valid calendar date", + }, + ) from error + + +def _window(start: Optional[str], end: Optional[str]) -> DateWindow: + window = DateWindow(start=_parse_day(start, "start"), end=_parse_day(end, "end")) + if window.start is not None and window.end is not None and window.start > window.end: + raise HTTPException( + status_code=422, + detail={ + "code": "INVALID_WINDOW", + "message": "start must not be after end", + }, + ) + return window + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _mark_reassignable(db: Any, study_id: uuid.UUID, body: dict[str, Any]) -> None: + """Flag the rows whose arm the owner may still set by hand. + + Only studies whose frozen policy allows manual assignment, only active + enrollments, and only before any activity (the same guard the override + route enforces, computed with one study-wide query). + """ + row = db.get(StudyRow, study_id) + config = getattr(row, "research_config_json", None) + allowed = ( + row is not None + and manual_override_enabled(config) + and getattr(row, "research_status", None) != ResearchStudyStatus.STUDY_STOPPED.value + ) + used = {str(item) for item in enrollments_with_activity(db, study_id)} if allowed else set() + for participant in body.get("participants") or []: + participant["reassignable"] = bool( + allowed + and participant.get("status") == "ACTIVE" + and participant.get("arm") is not None + and participant.get("enrollment_id") not in used + ) + + +@router.get( + "/{study_id}/analytics/participants", + summary="Participants table with arm and activity (study owner/admin only)", +) +def study_participants_analytics( + study_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + _authorize(db, current_user, study_id) + body = analytics.build_participants( + analytics_store.load_study_frame(db, study_id), + analytics_store.load_events(db, study_id), + analytics_store.load_daily_event_counts(db, study_id), + study_id=str(study_id), + now=_now(), + budgets={ + str(view.enrollment_id): budget_ledger.balance_summary(view) + for view in budget_ledger.list_study_balances(db, study_id) + }, + ) + _mark_reassignable(db, study_id, body) + return JsonResponseWithStatus(status_code=200, content=body) + finally: + db.close() + + +@router.get( + "/{study_id}/analytics/participants/{enrollment_id}", + summary="One participant's telemetry dashboard (study owner/admin only)", +) +def study_participant_dashboard( + study_id: uuid.UUID, + enrollment_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + _authorize(db, current_user, study_id) + if analytics_store.enrollment_study_id(db, enrollment_id) != study_id: + raise _enrollment_not_found() + body = analytics.build_participant_detail( + analytics_store.load_study_frame(db, study_id, enrollment_id=enrollment_id), + str(enrollment_id), + analytics_store.load_events(db, study_id, enrollment_id=enrollment_id), + analytics_store.load_daily_event_counts( + db, study_id, enrollment_id=enrollment_id + ), + # The metrics layer drops permission rows nobody answered before it + # cuts to MAX_TIMELINE, so read a margin or the page shows fewer. + analytics_store.load_timeline(db, study_id, enrollment_id, limit=analytics.MAX_TIMELINE * 2), + study_id=str(study_id), + now=_now(), + ) + if body is None: + raise _enrollment_not_found() + return JsonResponseWithStatus(status_code=200, content=body) + finally: + db.close() + + +@router.get( + "/{study_id}/analytics/summary", + summary="Study totals and arm comparison (study owner/admin only)", +) +def study_analytics_summary( + study_id: uuid.UUID, + start: Optional[str] = Query(None, description="Inclusive UTC start date (YYYY-MM-DD)"), + end: Optional[str] = Query(None, description="Inclusive UTC end date (YYYY-MM-DD)"), + arm: Optional[list[uuid.UUID]] = Query(None, description="Only these arms (profile ids); repeatable"), + participant: Optional[list[uuid.UUID]] = Query( + None, description="Only these participants (enrollment ids); repeatable" + ), + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + _authorize(db, current_user, study_id) + window = _window(start, end) + full_frame = analytics_store.load_study_frame(db, study_id) + try: + frame, enrollment_ids, applied = analytics.resolve_filters(full_frame, arm, participant) + except analytics.FilterError as error: + raise HTTPException( + status_code=422, detail={"code": error.code, "message": str(error)} + ) from error + # A filter that matches nobody needs no event scan. + scan = enrollment_ids is None or bool(enrollment_ids) + scoped = [uuid.UUID(item) for item in enrollment_ids] if enrollment_ids is not None else None + body = analytics.build_study_summary( + frame, + analytics_store.load_events(db, study_id, enrollment_ids=scoped, window=window) if scan else [], + ( + analytics_store.load_daily_event_counts( + db, study_id, enrollment_ids=scoped, window=window + ) + if scan + else [] + ), + study_id=str(study_id), + now=_now(), + window=window, + spend=budget_ledger.study_spend_summary(db, study_id, enrollment_ids=scoped), + ) + body["filters"] = applied + body["filter_options"] = analytics.filter_options(full_frame) + return JsonResponseWithStatus(status_code=200, content=body) + finally: + db.close() diff --git a/src/backend/routers/research/study_export.py b/src/backend/routers/research/study_export.py new file mode 100644 index 0000000..a76a445 --- /dev/null +++ b/src/backend/routers/research/study_export.py @@ -0,0 +1,377 @@ +"""Owner-only raw data export of a study (ZIP of CSV or JSONL files). + +Participants appear only by enrollment id and study-local code; retention +tombstones are never exported. Content (prompts, reasoning, tool data) is +included only on request, only for a study that captured it, and only in JSONL. +Every export is logged. The whole ZIP is built from one database snapshot +before it is streamed. +""" + +from __future__ import annotations + +import logging +import re +import uuid # noqa: TC003 - FastAPI resolves the path-parameter annotations at runtime +from collections import defaultdict +from datetime import datetime, timezone +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi.responses import StreamingResponse +from sqlalchemy import select + +from App import App +from backend.routers.analytics.auth_utils import AuthenticatedUser, get_current_user +from backend.routers.research.study_analytics import _window +from database.db_schemas import Study as StudyRow +from database.research_schemas import StudyAgentProfile +from research.analysis.study_analytics import metrics as analytics +from research.analysis.study_analytics import store as analytics_store +from research.analysis.study_export.build import ( + CHAT_COLUMNS, + DATASETS, + EVENT_CSV_COLUMNS, + EVENT_JSONL_COLUMNS, + FORMATS, + PARTICIPANT_COLUMNS, + SESSION_COLUMNS, + ExportWriter, + dictionary, + event_csv_row, + export_envelope, + metric_columns, + optional_iso, +) +from research.analysis.study_export.store import EVENT_CATEGORIES, iter_export_events +from research.runtime.assignment.hashing import ( + assignment_strategy, + hash_arm_index, + manual_override_enabled, +) +from research.study.protocol import store as study_store +from research.study.protocol.enums import AssignmentStrategy + +router = APIRouter() + +EXPORT_VERSION = 1 +ASSIGNMENT_FORMULA = ( + "arm index k = floor(K * h / 2^64), h = first 8 bytes (big-endian) of " + "SHA-256('{study_id}:{randomization_epoch}:{enrollment_id}'), over the arms ordered by selection_order " + "(UUIDs lowercase and hyphenated, randomization_epoch 0)" +) + + +def _unprocessable(code: str, message: str) -> HTTPException: + return HTTPException(status_code=422, detail={"code": code, "message": message}) + + +def _filename(name: str, now: datetime) -> str: + slug = re.sub(r"[^A-Za-z0-9._-]+", "-", name or "").strip("-")[:60] or "study" + return f"{slug}-export-{now.strftime('%Y%m%d-%H%M')}.zip" + + +def _stream(file: Any): + try: + while chunk := file.read(1024 * 1024): + yield chunk + finally: + file.close() + + +@router.get( + "/{study_id}/export", + summary="Export study data as a ZIP of CSV/JSONL files with a manifest (owner/admin)", +) +def export_study_data( + study_id: uuid.UUID, + datasets: Optional[list[str]] = Query(None, description=f"Repeatable; any of {', '.join(DATASETS)} (default all)"), + export_format: str = Query("csv", alias="format", description="csv or jsonl"), + arm: Optional[list[uuid.UUID]] = Query(None, description="Only these arms (profile ids); repeatable"), + participant: Optional[list[uuid.UUID]] = Query(None, description="Only these participants; repeatable"), + start: Optional[str] = Query(None, description="Inclusive UTC start date (YYYY-MM-DD) for events"), + end: Optional[str] = Query(None, description="Inclusive UTC end date (YYYY-MM-DD) for events"), + event_categories: Optional[list[str]] = Query( + None, description=f"Repeatable; any of {', '.join([*EVENT_CATEGORIES, 'other'])} (default all)" + ), + include_content: bool = Query(False, description="Include captured content (JSONL only)"), + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + selected = list(dict.fromkeys(datasets or DATASETS)) + unknown = [name for name in selected if name not in DATASETS] + if unknown: + raise _unprocessable("UNKNOWN_DATASET", f"unknown datasets: {', '.join(unknown)}") + if export_format not in FORMATS: + raise _unprocessable("UNKNOWN_FORMAT", "format must be csv or jsonl") + categories = list(dict.fromkeys(event_categories or [])) + unknown = [name for name in categories if name not in EVENT_CATEGORIES and name != "other"] + if unknown: + raise _unprocessable("UNKNOWN_EVENT_CATEGORY", f"unknown event categories: {', '.join(unknown)}") + window = _window(start, end) + + db = app.get_db_session() + try: + # One snapshot for every dataset of the export. + db.connection(execution_options={"isolation_level": "REPEATABLE READ"}) + from backend.routers.research.access import require_study_owner + + study = study_store.get_study(db, study_id) + if study is None: + raise HTTPException(status_code=404, detail="Study not found") + require_study_owner(current_user, study) + study_row = db.get(StudyRow, study_id) + config = getattr(study_row, "research_config_json", None) or {} + policy = config.get("telemetry_policy") if isinstance(config.get("telemetry_policy"), dict) else {} + content_captured = policy.get("content_capture") is True + if include_content and not content_captured: + raise _unprocessable("CONTENT_NOT_CAPTURED", "this study does not capture content") + if include_content and export_format != "jsonl": + raise _unprocessable("CONTENT_REQUIRES_JSONL", "content is exported in the JSONL format only") + + full_frame = analytics_store.load_study_frame(db, study_id) + try: + frame, enrollment_ids, applied = analytics.resolve_filters(full_frame, arm, participant) + except analytics.FilterError as error: + raise _unprocessable(error.code, str(error)) from error + scoped = [uuid.UUID(item) for item in enrollment_ids] if enrollment_ids is not None else None + scan = scoped is None or bool(scoped) + codes = {row.enrollment_id: row.participant_code for row in full_frame.enrollments} + assignments = {row.enrollment_id: row for row in full_frame.assignments} + arm_of = {row.enrollment_id: row.profile_id for row in full_frame.assignments} + enrollments = sorted(frame.enrollments, key=lambda row: row.participant_code) + strategy = assignment_strategy(config) + arm_rows = db.execute( + select( + StudyAgentProfile.profile_id, + StudyAgentProfile.selection_order, + StudyAgentProfile.profile_digest, + ) + .where(StudyAgentProfile.study_id == study_id) + .order_by(StudyAgentProfile.selection_order.asc(), StudyAgentProfile.profile_id.asc()) + ).all() + ordered_arms = [str(profile_id) for profile_id, _, _ in arm_rows] + + writer = ExportWriter(export_format) + columns: dict[str, Any] = {} + + analyses: dict[str, Any] = {} + if {"participant_metrics", "chats"} & set(selected): + events = ( + analytics_store.load_events(db, study_id, enrollment_ids=scoped, window=window) + if scan + else [] + ) + # Presence and IDE edits come from the per-day aggregates over every + # retained event, exactly as the analytics summary computes them. + daily = ( + analytics_store.load_daily_event_counts(db, study_id, enrollment_ids=scoped, window=window) + if scan + else [] + ) + events_by_enrollment: dict[str, list] = defaultdict(list) + for row in events: + events_by_enrollment[row.enrollment_id].append(row) + daily_by_enrollment: dict[str, list] = defaultdict(list) + for row in daily: + daily_by_enrollment[row.enrollment_id].append(row) + sessions_by_enrollment: dict[str, list] = defaultdict(list) + for row in frame.sessions: + sessions_by_enrollment[row.enrollment_id].append(row) + for enrollment in enrollments: + analyses[enrollment.enrollment_id] = analytics.analyze_participant( + events_by_enrollment.get(enrollment.enrollment_id, ()), + sessions_by_enrollment.get(enrollment.enrollment_id, ()), + daily_by_enrollment.get(enrollment.enrollment_id) or None, + window=window, + ) + + if "participants" in selected: + def randomized(enrollment_id: str) -> Optional[str]: + if strategy != AssignmentStrategy.DETERMINISTIC_HASH.value or not ordered_arms: + return None + return ordered_arms[hash_arm_index(study_id, 0, enrollment_id, len(ordered_arms))] + + columns["participants"] = PARTICIPANT_COLUMNS + writer.write_rows( + "participants", + PARTICIPANT_COLUMNS, + ( + { + "enrollment_id": row.enrollment_id, + "participant_code": row.participant_code, + "status": row.status, + "enrolled_at": row.enrolled_at, + "consent_accepted_at": row.consent_accepted_at, + "consent_digest": row.consent_digest, + "consent_answers": row.consent_answers, + "arm_profile_id": getattr(assignments.get(row.enrollment_id), "profile_id", None), + "arm_name": getattr(assignments.get(row.enrollment_id), "name", None), + "arm_model": getattr(assignments.get(row.enrollment_id), "model", None), + "arm_runtime": getattr(assignments.get(row.enrollment_id), "framework_version", None), + "assignment_strategy": getattr(assignments.get(row.enrollment_id), "strategy", None), + "assigned_at": getattr(assignments.get(row.enrollment_id), "assigned_at", None), + "randomized_profile_id": randomized(row.enrollment_id), + } + for row in enrollments + ), + ) + if "participant_metrics" in selected: + metric_spec = metric_columns() + columns["participant_metrics"] = metric_spec + writer.write_rows( + "participant_metrics", + metric_spec, + ( + { + "enrollment_id": row.enrollment_id, + "participant_code": row.participant_code, + "arm_profile_id": arm_of.get(row.enrollment_id), + "has_telemetry": analyses[row.enrollment_id].has_telemetry, + **analyses[row.enrollment_id].metrics(), + } + for row in enrollments + ), + ) + if "sessions" in selected: + columns["sessions"] = SESSION_COLUMNS + writer.write_rows( + "sessions", + SESSION_COLUMNS, + ( + { + "session_id": row.session_id, + "enrollment_id": row.enrollment_id, + "participant_code": codes.get(row.enrollment_id), + "state": row.state, + "opened_at": row.opened_at, + "closed_at": row.closed_at, + "last_activity_at": row.last_activity_at, + "last_heartbeat_at": row.last_heartbeat_at, + "close_reason": row.close_reason, + "session_seconds": round(analytics.session_seconds(row, window), 1), + } + for row in sorted(frame.sessions, key=lambda item: (item.opened_at is None, item.opened_at or datetime.min.replace(tzinfo=timezone.utc))) + ), + ) + if "chats" in selected: + columns["chats"] = CHAT_COLUMNS + writer.write_rows( + "chats", + CHAT_COLUMNS, + ( + { + "enrollment_id": row.enrollment_id, + "participant_code": row.participant_code, + "arm_profile_id": arm_of.get(row.enrollment_id), + **chat, + } + for row in enrollments + for chat in analytics.chat_rows(analyses[row.enrollment_id].chats, limit=10**9) + ), + ) + if "events" in selected: + event_spec = EVENT_JSONL_COLUMNS if export_format == "jsonl" else EVENT_CSV_COLUMNS + + def event_rows(): + if not scan: + return + for ( + event_id, + occurred_at, + enrollment_id, + research_session_id, + event_type, + source, + emitter_id, + emitter_sequence, + envelope, + ) in iter_export_events( + db, study_id, enrollment_ids=scoped, window=window, categories=categories or None + ): + base = { + "event_id": event_id, + "occurred_at": occurred_at, + "enrollment_id": enrollment_id, + "participant_code": codes.get(enrollment_id), + "arm_profile_id": arm_of.get(enrollment_id), + "research_session_id": research_session_id, + "event_type": event_type, + "source": source, + } + envelope = envelope if isinstance(envelope, dict) else {} + if export_format == "jsonl": + yield {**base, "envelope": export_envelope(envelope, include_content=include_content)} + else: + yield { + **base, + "emitter_id": emitter_id, + "emitter_sequence": emitter_sequence, + **event_csv_row(envelope), + } + + columns["events"] = event_spec + writer.write_rows("events", event_spec, event_rows()) + + now = datetime.now(timezone.utc) + writer.write_json( + "manifest.json", + { + "export_version": EXPORT_VERSION, + "generated_at": now.isoformat(), + "study": { + "study_id": str(study_id), + "name": getattr(study_row, "name", None), + "research_status": getattr(study_row, "research_status", None), + "created_at": optional_iso(getattr(study_row, "created_at", None)), + "research_config_digest": getattr(study_row, "research_config_digest", None), + }, + "telemetry_policy": { + "content_capture": content_captured, + "allowed_field_classes": list(policy.get("allowed_field_classes") or []), + }, + "assignment": { + "strategy": strategy, + "manual_override": manual_override_enabled(config), + "formula": ASSIGNMENT_FORMULA if strategy == AssignmentStrategy.DETERMINISTIC_HASH.value else None, + "arms": [ + {"index": index, "profile_id": str(profile_id), "selection_order": order, "profile_digest": digest} + for index, (profile_id, order, digest) in enumerate(arm_rows) + ], + }, + "filters": { + "datasets": selected, + "format": export_format, + "arms": applied["arms"], + "participants": applied["participants"], + "start": start, + "end": end, + "event_categories": categories or "all", + "include_content": include_content, + }, + "files": writer.counts, + "columns": dictionary(columns), + "notes": [ + "Participants are identified only by enrollment id and study-local participant code.", + "Events deleted by retention or erasure are not included.", + "Metrics and chats are computed over the exported date range, like the analytics page.", + ], + }, + ) + logging.info( + "[Research/export] user %s exported study=%s datasets=%s format=%s include_content=%s files=%s", + current_user.user_id, + study_id, + ",".join(selected), + export_format, + include_content, + writer.counts, + ) + archive = writer.finish() + filename = _filename(getattr(study_row, "name", ""), now) + finally: + db.close() + return StreamingResponse( + _stream(archive), + media_type="application/zip", + headers={"Content-Disposition": f'attachment; filename="{filename}"', "Cache-Control": "no-store"}, + ) diff --git a/src/backend/routers/research/study_traces.py b/src/backend/routers/research/study_traces.py new file mode 100644 index 0000000..002680f --- /dev/null +++ b/src/backend/routers/research/study_traces.py @@ -0,0 +1,141 @@ +"""Owner-only trace of one participant chat (prompts, reasoning, tool calls). + +Separate from the analytics router because a trace returns stored content: +what the study's telemetry policy and the participant's consent kept, with +``[REDACTED]`` values reported as not captured. Study owner or administrator +only, never cached, and every read is logged. +""" + +from __future__ import annotations + +import base64 +import binascii +import json +import logging +import uuid # noqa: TC003 - FastAPI resolves the path-parameter annotations at runtime +from datetime import datetime +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query + +from App import App +from backend.Responses import JsonResponseWithStatus +from backend.routers.analytics.auth_utils import AuthenticatedUser, get_current_user +from database.db_schemas import Study as StudyRow +from database.research_schemas import ResearchEnrollment +from research.analysis.study_analytics import store as analytics_store +from research.analysis.study_traces.assemble import build_page +from research.analysis.study_traces.store import iter_chat_events +from research.study.protocol import store as study_store + +router = APIRouter() + +MAX_TURNS_PER_PAGE = 100 + + +def _authorize(db: Any, current_user: AuthenticatedUser, study_id: uuid.UUID) -> None: + from backend.routers.research.access import require_study_owner + + study = study_store.get_study(db, study_id) + if study is None: + raise HTTPException(status_code=404, detail="Study not found") + require_study_owner(current_user, study) + + +def _invalid_cursor() -> HTTPException: + return HTTPException( + status_code=422, + detail={"code": "INVALID_CURSOR", "message": "the page cursor is not valid"}, + ) + + +def encode_cursor(row: Any, next_index: int) -> str: + occurred_at, emitter_id, emitter_sequence, event_id = row.key + raw = json.dumps( + {"t": occurred_at.isoformat(), "e": emitter_id, "s": emitter_sequence, "i": event_id, "n": next_index}, + separators=(",", ":"), + ) + return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=") + + +def decode_cursor(cursor: str) -> tuple[tuple[datetime, str, int, str], int]: + """``(position of the page's first event, turns before the page)``.""" + try: + padded = cursor + "=" * (-len(cursor) % 4) + data = json.loads(base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8")) + position = ( + datetime.fromisoformat(data["t"]), + str(data["e"]), + int(data["s"]), + str(uuid.UUID(str(data["i"]))), + ) + next_index = int(data["n"]) + except (ValueError, KeyError, TypeError, binascii.Error, UnicodeError) as error: + raise _invalid_cursor() from error + if next_index < 0: + raise _invalid_cursor() + return position, next_index + + +@router.get( + "/{study_id}/enrollments/{enrollment_id}/trace", + summary="One participant chat as turns of prompts, reasoning, messages and tool calls (owner/admin)", +) +def study_chat_trace( + study_id: uuid.UUID, + enrollment_id: uuid.UUID, + chat_id: str = Query(..., min_length=1, max_length=256, description="The ACP chat (session) id"), + cursor: Optional[str] = Query(None, max_length=1024), + limit: int = Query(20, ge=1, le=MAX_TURNS_PER_PAGE, description="Turns per page"), + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + db = app.get_db_session() + try: + _authorize(db, current_user, study_id) + if analytics_store.enrollment_study_id(db, enrollment_id) != study_id: + raise HTTPException( + status_code=404, + detail={"code": "ENROLLMENT_NOT_FOUND", "message": "no such enrollment in this study"}, + ) + start_index = 0 + position = None + if cursor: + position, start_index = decode_cursor(cursor) + turns, next_row, next_index = build_page( + iter_chat_events(db, study_id, enrollment_id, chat_id, start_at=position), + limit=limit, + start_index=start_index, + ) + if not cursor and not turns: + raise HTTPException( + status_code=404, + detail={"code": "CHAT_NOT_FOUND", "message": "no events of this chat in this enrollment"}, + ) + study_row = db.get(StudyRow, study_id) + policy = ((getattr(study_row, "research_config_json", None) or {}).get("telemetry_policy")) or {} + participant_code = db.get(ResearchEnrollment, enrollment_id).participant_code + logging.info( + "[Research/trace] user %s read chat trace study=%s enrollment=%s turns=%d", + current_user.user_id, + study_id, + enrollment_id, + len(turns), + ) + response = JsonResponseWithStatus( + status_code=200, + content={ + "study_id": str(study_id), + "enrollment_id": str(enrollment_id), + "participant_code": participant_code, + "chat_id": chat_id, + "content_capture_enabled": isinstance(policy, dict) and policy.get("content_capture") is True, + "turns": turns, + "next_cursor": encode_cursor(next_row, next_index) if next_row is not None else None, + "has_more": next_row is not None, + }, + ) + response.headers["Cache-Control"] = "no-store" + return response + finally: + db.close() diff --git a/src/backend/routers/research/telemetry.py b/src/backend/routers/research/telemetry.py new file mode 100644 index 0000000..8a2aebf --- /dev/null +++ b/src/backend/routers/research/telemetry.py @@ -0,0 +1,316 @@ +"""Telemetry ingestion API (Issue 09). + +Mounted under ``/api/research/telemetry``. Batch submission is authenticated +solely by the scoped session capability carried in the request body — a normal +account/session cookie is never accepted in its place. Receipt inspection is +admin/researcher-only. +""" + +from __future__ import annotations + +import logging +import os +import uuid # noqa: TC003 - FastAPI evaluates route annotations at runtime +from datetime import datetime, timezone +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.exc import SQLAlchemyError + +from App import App +from backend.Responses import JsonResponseWithStatus +from database.db_schemas import ResearchStudyStatus, Study as StudyRow +from privacy import collection +from research.participants.enums import EnrollmentStatus +from backend.routers.analytics.auth_utils import ( + AuthenticatedUser, + get_current_user, + require_admin, +) +from backend.routers.research import bootstrap as bootstrap_router +from backend.routers.research.bootstrap import BOOTSTRAP_SIGNING_SECRET +from research.analysis.operations import store as operations_store +from research.participants import identity as identity_store +from research.runtime.bootstrap.capability import verify_capability +from research.runtime.bootstrap.models import ( + CapabilityVerification, # noqa: TC001 - FastAPI evaluates route annotations at runtime + SessionCapability, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) +from research.runtime.sessions import store as session_store +from research.telemetry.ingestion.agent_tasks import ensure_agent_tasks_for_ack +from research.telemetry.ingestion.models import ( + TelemetryBatchAckV1, # noqa: TC001 - FastAPI evaluates route annotations at runtime + TelemetryBatchRequestV1, # noqa: TC001 - FastAPI evaluates route annotations at runtime +) +from research.telemetry.ingestion.service import ingest_batch +from research.telemetry.privacy import PrivacyPolicy +from research.telemetry.ingestion.store import SqlAlchemyIngestionStore, get_receipt_by_id + +router = APIRouter() + +logger = logging.getLogger(__name__) + +AUDIENCE = "research-runtime" +SCOPE_WRITE = "telemetry:write" + +# Events of an already ENDED session are still stored while they occurred no +# later than this many seconds after the server closed it (C-02): the last poll +# window, the closing events and an offline tail of an idle-expired session. +LATE_EVENT_GRACE_SECONDS = int(os.environ.get("TELEMETRY_LATE_EVENT_GRACE_SECONDS", "900")) + + +def _signing_secret() -> str: + """The signing secret at request time. + + Delegates to the bootstrap router's resolver so the signer and every + verifier agree (an installed test signer wins there too); this module's + import-time copy is only a patch target of last resort. + """ + return bootstrap_router.signing_secret() or BOOTSTRAP_SIGNING_SECRET or "" + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _log_ack(ack: TelemetryBatchAckV1) -> None: + """Emit exactly one compact line per batch so stalled uploads are visible. + + Records only ids, counts and the distinct reason codes — never event + payloads — so a wedged spool (`docker logs backend`) is diagnosable. + """ + rejected_reasons = sorted( + {event.reason.value for event in ack.rejected if event.reason is not None} + ) + retryable_reasons = sorted( + {event.reason.value for event in ack.retryable if event.reason is not None} + ) + log = logger.warning if (ack.rejected or ack.retryable) else logger.info + log( + "telemetry batch %s ack: accepted=%d duplicate=%d rejected=%d retryable=%d " + "rejected_reasons=%s retryable_reasons=%s", + ack.batch_id, + len(ack.accepted), + len(ack.duplicate), + len(ack.rejected), + len(ack.retryable), + rejected_reasons or "-", + retryable_reasons or "-", + ) + + +def _verify_session_capability( + capability: SessionCapability, + *, + now: datetime, + current_revocation_epoch: int, + expected_enrollment_id: Optional[uuid.UUID] = None, + expected_research_session_id: Optional[uuid.UUID] = None, + expected_study_id: Optional[uuid.UUID] = None, +) -> CapabilityVerification: + """Verify the batch capability against the subject resolved from its events.""" + return verify_capability( + capability, + _signing_secret(), + expected_audience=AUDIENCE, + expected_scope=[SCOPE_WRITE], + now=now, + current_revocation_epoch=current_revocation_epoch, + expected_enrollment_id=expected_enrollment_id, + expected_research_session_id=expected_research_session_id, + expected_study_id=expected_study_id, + ) + + +def _verify_receipt_capability( + capability: SessionCapability, + *, + now: datetime, + current_revocation_epoch: Optional[int], + expected_enrollment_id: Optional[uuid.UUID] = None, + expected_research_session_id: Optional[uuid.UUID] = None, + expected_study_id: Optional[uuid.UUID] = None, +) -> CapabilityVerification: + """Verify a capability for an *already committed* receipt (ISSUE-08). + + Re-reading an immutable ACK is not new data collection, so expiry and the + live revocation epoch must not block a producer from confirming a batch the + server already accepted. The signature, audience, scope and subject binding + are still mandatory: knowing a ``batch_id`` is never sufficient. + """ + return verify_capability( + capability, + _signing_secret(), + expected_audience=AUDIENCE, + expected_scope=[SCOPE_WRITE], + now=capability.issued_at, + current_revocation_epoch=None, + expected_enrollment_id=expected_enrollment_id, + expected_research_session_id=expected_research_session_id, + expected_study_id=expected_study_id, + ) + + +def _ingestion_kill_switch_check(db, payload: TelemetryBatchRequestV1): + """DB-backed kill-switch predicate scoped to the batch's active subject. + + The batch is anchored by its first in-scope event (matching the ingestion + service's anchor rule), so the check blocks exactly the study/revision/ + enrollment the accepted events would be attributed to. + """ + for event in payload.events: + if ( + event.enrollment_id is None + or event.research_session_id is None + ): + continue + session_row = session_store.get_session(db, event.research_session_id) + if session_row is None: + continue + enrollment_row = identity_store.get_enrollment(db, event.enrollment_id) + if enrollment_row is None: + continue + study_id = session_row.study_id + return operations_store.db_kill_switch_check( + db, + study_id=study_id, + enrollment_id=event.enrollment_id, + ) + + def _never_engaged() -> bool: + return False + + return _never_engaged + + +def _enrollment_resolver(db): + def resolve(enrollment_id: uuid.UUID): + # Lock the enrollment row so concurrent batches for the same subject + # serialize inside the ingestion transaction. + row = identity_store.get_enrollment(db, enrollment_id, for_update=True) + if row is None: + return None + enrollment = identity_store.row_to_enrollment(row) + study = db.get(StudyRow, row.study_id) + # A stopped study refuses an active enrollment's events. Only this answer + # says so: the stored status is the stop cascade's to write, and the + # ingestion commit must never turn a revoked or completed one into it. + if ( + getattr(study, "research_status", None) == ResearchStudyStatus.STUDY_STOPPED.value + and enrollment.status == EnrollmentStatus.ACTIVE + ): + enrollment = enrollment.model_copy(update={"status": EnrollmentStatus.STUDY_STOPPED}) + return enrollment + + return resolve + + +def _session_resolver(db): + def resolve(research_session_id: uuid.UUID): + row = session_store.get_session(db, research_session_id, for_update=True) + return session_store.row_to_session(row) if row is not None else None + + return resolve + + +def _privacy_policy_resolver(db): + def resolve(enrollment): + study = db.get(StudyRow, enrollment.study_id) + if study is None: + raise HTTPException(status_code=409, detail="Study policy is unavailable") + config = study.research_config_json or {} + return PrivacyPolicy.from_study_policy( + config.get("telemetry_policy") or {}, + consent_active=( + enrollment.status == EnrollmentStatus.ACTIVE + and enrollment.consent_accepted_at is not None + ), + ) + + return resolve + + +@router.post("/batches", summary="Submit a canonical telemetry batch") +def submit_telemetry_batch( + payload: TelemetryBatchRequestV1, + app: App = Depends(App.get_instance), +): + """Ingest a batch authenticated by its scoped session capability.""" + if not payload.events: + # No subject to authorize and no fact to store: refuse at the boundary + # with a typed reason and never create a durable receipt (ISSUE-08). + raise HTTPException( + status_code=422, + detail={ + "code": "EMPTY_BATCH", + "message": "a telemetry batch must contain at least one event", + }, + ) + db = app.get_db_session() + try: + store = SqlAlchemyIngestionStore(db) + ack = ingest_batch( + payload, + capability_verifier=_verify_session_capability, + receipt_capability_verifier=_verify_receipt_capability, + enrollment_resolver=_enrollment_resolver(db), + session_resolver=_session_resolver(db), + store=store, + now=_now(), + kill_switch_check=_ingestion_kill_switch_check(db, payload), + privacy_policy_resolver=_privacy_policy_resolver(db), + late_event_grace_seconds=LATE_EVENT_GRACE_SECONDS, + ) + _log_ack(ack) + # ACP-proxied runs carry an agent_run_id but no task: materialize the + # linkable agent_task the dashboards join on. This is idempotent and + # never changes the ack or the accepted facts. + ensure_agent_tasks_for_ack( + db, + payload, + ack, + before_create=lambda account_id: collection.lock_account(db, account_id), + ) + return JsonResponseWithStatus( + status_code=200, content=ack.model_dump(mode="json") + ) + except SQLAlchemyError as error: + db.rollback() + raise HTTPException( + status_code=503, detail="telemetry store is unavailable" + ) from error + finally: + db.close() + + +@router.get("/receipts/{receipt_id}", summary="Fetch an immutable batch receipt") +def get_batch_receipt( + receipt_id: uuid.UUID, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +): + """Admin/researcher view of a stored batch receipt.""" + require_admin(current_user) + db = app.get_db_session() + try: + receipt = get_receipt_by_id(db, receipt_id) + if receipt is None: + raise HTTPException(status_code=404, detail="Batch receipt not found") + return JsonResponseWithStatus( + status_code=200, + content={ + "receipt": receipt.ack.model_dump(mode="json"), + "enrollment_id": ( + str(receipt.enrollment_id) + if receipt.enrollment_id is not None + else None + ), + "research_session_id": ( + str(receipt.research_session_id) + if receipt.research_session_id is not None + else None + ), + }, + ) + finally: + db.close() diff --git a/src/backend/routers/session/acquire.py b/src/backend/routers/session/acquire.py index 3d14cdd..864cb96 100644 --- a/src/backend/routers/session/acquire.py +++ b/src/backend/routers/session/acquire.py @@ -80,15 +80,14 @@ def acquire_session( ) user_info = redis_manager.get("user_token", user_id) - session_token = None - # Create a new session token if none exists or invalid + session_token = (user_info or {}).get("session_token") + # Reuse the account's session only while it is live in Redis, not merely + # recorded in the database (a dead one fails every project call); reusing + # it restarts its expiry. Create a new session token otherwise. if ( - not user_info - or not user_info.get("session_token") - or crud.get_session_by_id( - db_session, uuid.UUID(user_info.get("session_token")) - ) - is None + not session_token + or crud.get_session_by_id(db_session, uuid.UUID(session_token)) is None + or not redis_manager.touch("session_token", session_token) ): session_token = create_uuid() crud.create_session( @@ -97,17 +96,15 @@ def acquire_session( session_token, ) - # Update session token in Redis with new session token while preserving TTL - redis_manager.set("user_token", user_id, {"session_token": session_token}) - - # Store session token entry separately + # The session entry first: the account's link never names a missing one. redis_manager.set( "session_token", session_token, {"user_token": user_id, "project_tokens": []}, ) + redis_manager.set("user_token", user_id, {"session_token": session_token}) else: - session_token = user_info.get("session_token") + redis_manager.touch("user_token", user_id) response_obj = JsonResponseWithStatus( status_code=200, @@ -118,6 +115,7 @@ def acquire_session( value=session_token, httponly=True, samesite="lax", + secure=app.get_config().cookie_secure, expires=config.session_token_expires_in_seconds, ) return response_obj diff --git a/src/backend/routers/session/deactivate.py b/src/backend/routers/session/deactivate.py index 2df1298..fe456df 100644 --- a/src/backend/routers/session/deactivate.py +++ b/src/backend/routers/session/deactivate.py @@ -1,19 +1,16 @@ import logging import uuid -from typing import Union from fastapi import APIRouter, Cookie, Depends +from agents import lifecycle from App import App from backend.Responses import ( DeactivateSessionError, DeactivateSessionPostResponse, ErrorResponse, InvalidOrExpiredAuthToken, - InvalidOrExpiredSessionToken, JsonResponseWithStatus, - SessionNotFoundError, - UserNotFoundError, ) from database import crud @@ -22,12 +19,11 @@ @router.put( - "/", + "", response_model=None, responses={ "200": {"model": DeactivateSessionPostResponse}, - "401": {"model": InvalidOrExpiredSessionToken}, - "404": {"model": Union[UserNotFoundError, SessionNotFoundError]}, + "401": {"model": InvalidOrExpiredAuthToken}, "422": {"model": ErrorResponse}, "429": {"model": ErrorResponse}, "500": {"model": DeactivateSessionError}, @@ -63,36 +59,27 @@ def deactivate_session( ) user_id = auth_info["user_id"] - if crud.get_user_by_id(db_session, uuid.UUID(user_id)) is None: - return JsonResponseWithStatus( - status_code=404, - content=UserNotFoundError(), - ) - user_info = redis_manager.get("user_token", user_id) - if user_info is None or not user_info.get("session_token"): - return JsonResponseWithStatus( - status_code=401, - content=InvalidOrExpiredSessionToken(), - ) + session_token = (user_info or {}).get("session_token") + # Close any agent tasks this session left open before tearing it down. + # The plugin closes the *previous* task on startup, but it can't detect + # logout, so without this a task ends its life stuck in 'running' with + # no aggregated totals. Failures here are logged and ignored: telemetry + # bookkeeping must not block the user from logging out. + if session_token: + session_id = uuid.UUID(session_token) + if crud.get_session_by_id(db_session, session_id) is not None: + for task in crud.get_open_agent_tasks_for_session(db_session, session_id): + try: + lifecycle.finalize_agent_task(db_session, task.task_id) + except Exception as e: + logging.warning( + f"[Session/deactivate] failed to finalize agent task " + f"{task.task_id}: {e}" + ) - session_token = user_info.get("session_token") - if crud.get_session_by_id(db_session, uuid.UUID(session_token)) is None: - return JsonResponseWithStatus( - status_code=404, - content=SessionNotFoundError(), - ) - # Retrieve session info from Redis - session_info = redis_manager.get("session_token", session_token) - # Validate session token presence and validity - if session_info is None: - return JsonResponseWithStatus( - status_code=401, - content=InvalidOrExpiredSessionToken(), - ) - # Step 3: Delete the session from Redis and clear cookies - redis_manager.delete("user_token", user_id, db_session) - redis_manager.delete("session_token", session_token, db_session) + # Auth is the login credential; its Redis deletion also removes any + # linked session, but logout must work when no session was acquired. redis_manager.delete("auth_token", auth_token, db_session) # Create response and clear cookies @@ -107,6 +94,7 @@ def deactivate_session( value="", httponly=True, samesite="lax", + secure=app.get_config().cookie_secure, expires=-1, # Expire immediately ) @@ -115,7 +103,8 @@ def deactivate_session( key="session_token", value="", httponly=True, - samesite="lax", + samesite="lax", + secure=app.get_config().cookie_secure, expires=-1, # Expire immediately ) diff --git a/src/backend/routers/user/__init__.py b/src/backend/routers/user/__init__.py index 2ef2eb1..1740b4d 100644 --- a/src/backend/routers/user/__init__.py +++ b/src/backend/routers/user/__init__.py @@ -5,6 +5,7 @@ from .create import router as create_router from .delete import router as delete_router from .get import router as get_router +from .privacy import router as privacy_router from .reset_password import router as reset_password_router from .update import router as update_router from .verify import router as verify_router @@ -21,3 +22,4 @@ router.include_router(reset_password_router, prefix="/reset-password") router.include_router(get_router, prefix="/get") +router.include_router(privacy_router, prefix="/privacy") diff --git a/src/backend/routers/user/authenticate.py b/src/backend/routers/user/authenticate.py index b576e28..ab3f496 100644 --- a/src/backend/routers/user/authenticate.py +++ b/src/backend/routers/user/authenticate.py @@ -1,6 +1,5 @@ import logging import uuid -from typing import Union from fastapi import APIRouter, Depends @@ -10,16 +9,13 @@ from backend.Responses import ( AuthenticateUserError, AuthenticateUserNormalPostResponse, - AuthenticateUserOAuthPostResponse, AuthenticateUserPostResponse, ConfigNotFound, ErrorResponse, InvalidEmailOrPassword, - InvalidOrExpiredJWTToken, JsonResponseWithStatus, ) -from backend.utils import verify_jwt_token -from Queries import AuthenticateUserEmailPassword, AuthenticateUserOAuth +from Queries import AuthenticateUserEmailPassword from response_models import ResponseUser from utils import create_uuid @@ -63,7 +59,7 @@ def acquire_auth_token( response_model=AuthenticateUserPostResponse, responses={ "200": {"model": AuthenticateUserPostResponse}, - "401": {"model": Union[InvalidOrExpiredJWTToken, InvalidEmailOrPassword]}, + "401": {"model": InvalidEmailOrPassword}, "404": {"model": ConfigNotFound}, "422": {"model": ErrorResponse}, "429": {"model": ErrorResponse}, @@ -72,19 +68,18 @@ def acquire_auth_token( tags=["Authentication"], ) def authenticate_user( - user_to_authenticate: Union[AuthenticateUserEmailPassword, AuthenticateUserOAuth], + user_to_authenticate: AuthenticateUserEmailPassword, app: App = Depends(App.get_instance), ) -> JsonResponseWithStatus: """ - Authenticate a user using either OAuth (via JWT) or email/password. + Authenticate a user using email/password. This endpoint supports: - - OAuth: Validates a JWT token and fetches the user by email. - Email/Password: Verifies credentials against the database. Args: - user_to_authenticate (Union[AuthenticateUserEmailPassword, AuthenticateUserOAuth]): - Either an email/password object or a JWT-based OAuth object. + user_to_authenticate (AuthenticateUserEmailPassword): + An email/password object. app (App): FastAPI dependency that provides access to DB and config. Returns: @@ -96,92 +91,45 @@ def authenticate_user( config = app.get_config() try: - # OAuth Authentication - if isinstance(user_to_authenticate, AuthenticateUserOAuth): - verification_result = verify_jwt_token(user_to_authenticate.token) - if verification_result is None: - return JsonResponseWithStatus( - status_code=401, - content=InvalidOrExpiredJWTToken(), - ) - - found_user = crud.get_user_by_email( - db_session, verification_result["email"] + found_user = crud.get_user_by_email_password( + db_session, + str(user_to_authenticate.email), + user_to_authenticate.password.get_secret_value(), + ) + if not found_user: + return JsonResponseWithStatus( + status_code=401, + content=InvalidEmailOrPassword(), ) - if not found_user: - return JsonResponseWithStatus( - status_code=401, - content=InvalidOrExpiredJWTToken(), - ) - auth_token = acquire_auth_token(found_user.auth_token, found_user.user_id, redis_manager) # type: ignore - setattr(found_user, "auth_token", uuid.UUID(auth_token)) + auth_token = acquire_auth_token(found_user.auth_token, found_user.user_id, redis_manager) # type: ignore + setattr(found_user, "auth_token", uuid.UUID(auth_token)) - config_data = crud.get_config_by_id( - db_session, int(str(found_user.config_id)) - ) - if not config_data: - return JsonResponseWithStatus( - status_code=404, - content=ConfigNotFound(), - ) - - response_obj = JsonResponseWithStatus( - status_code=200, - content=AuthenticateUserOAuthPostResponse( - user=ResponseUser.model_validate(found_user), - config=str(config_data.config_data), - ), - ) - response_obj.set_cookie( - key="auth_token", - value=auth_token, - httponly=True, - samesite="lax", - expires=config.auth_token_expires_in_seconds, - ) - return response_obj - - # Email/Password Authentication - else: - found_user = crud.get_user_by_email_password( - db_session, - str(user_to_authenticate.email), - user_to_authenticate.password.get_secret_value(), + config_data = crud.get_config_by_id( + db_session, int(str(found_user.config_id)) + ) + if not config_data: + return JsonResponseWithStatus( + status_code=404, + content=ConfigNotFound(), ) - if not found_user: - return JsonResponseWithStatus( - status_code=401, - content=InvalidEmailOrPassword(), - ) - - auth_token = acquire_auth_token(found_user.auth_token, found_user.user_id, redis_manager) # type: ignore - setattr(found_user, "auth_token", uuid.UUID(auth_token)) - config_data = crud.get_config_by_id( - db_session, int(str(found_user.config_id)) - ) - if not config_data: - return JsonResponseWithStatus( - status_code=404, - content=ConfigNotFound(), - ) - - response_obj = JsonResponseWithStatus( - status_code=200, - content=AuthenticateUserNormalPostResponse( - user=ResponseUser.model_validate(found_user), - config=config_data.config_data, - ), - ) - response_obj.set_cookie( - key="auth_token", - value=auth_token, - httponly=True, - samesite="lax", - expires=config.auth_token_expires_in_seconds, - ) - return response_obj + response_obj = JsonResponseWithStatus( + status_code=200, + content=AuthenticateUserNormalPostResponse( + user=ResponseUser.model_validate(found_user), + config=config_data.config_data, + ), + ) + response_obj.set_cookie( + key="auth_token", + value=auth_token, + httponly=True, + samesite="lax", + secure=app.get_config().cookie_secure, + expires=config.auth_token_expires_in_seconds, + ) + return response_obj except Exception as e: logging.error(f"Error processing user authentication request: {str(e)}") diff --git a/src/backend/routers/user/create.py b/src/backend/routers/user/create.py index c1a1e91..440ccd4 100644 --- a/src/backend/routers/user/create.py +++ b/src/backend/routers/user/create.py @@ -2,17 +2,15 @@ This module defines a FastAPI router for user account creation. Endpoints: -- POST /: Creates a new user using either standard or OAuth registration. +- POST /: Creates a new user with email and password. Features: - Conflict checking for existing email addresses. -- Optional JWT verification for OAuth users. - Asynchronous email verification via Celery. - Structured response models and status codes. """ import logging -from typing import Union from fastapi import APIRouter, Depends @@ -23,11 +21,9 @@ CreateUserError, CreateUserPostResponse, ErrorResponse, - InvalidOrExpiredJWTToken, JsonResponseWithStatus, UserAlreadyExistsWithThisEmail, ) -from backend.utils import verify_jwt_token # Initialize the API router for user creation router = APIRouter() @@ -38,7 +34,6 @@ response_model=CreateUserPostResponse, responses={ "201": {"model": CreateUserPostResponse}, - "401": {"model": InvalidOrExpiredJWTToken}, "409": {"model": UserAlreadyExistsWithThisEmail}, "422": {"model": ErrorResponse}, "429": {"model": ErrorResponse}, @@ -46,15 +41,15 @@ }, ) def create_user( - user_to_create: Union[Queries.CreateUser, Queries.CreateUserOauth], + user_to_create: Queries.CreateUser, app: App = Depends(App.get_instance), ) -> JsonResponseWithStatus: """ - Create a new user in the system using standard or OAuth-based data. + Create a new user with email and password. Args: - user_to_create (Union[CreateUser, CreateUserOauth]): - User data from the request body (standard or OAuth-based). + user_to_create (CreateUser): + User data from the request body. app (App): Application context with access to database and services. @@ -63,10 +58,9 @@ def create_user( Flow: 1. Check if a user already exists with the given email. - 2. If using OAuth, validate the JWT token. - 3. Insert the new user into the database. - 4. Send a verification email via Celery. - 5. Return HTTP 201 with the new user ID. + 2. Insert the new user into the database. + 3. Send a verification email via Celery. + 4. Return HTTP 201 with the new user ID. """ db_session = app.get_db_session() @@ -79,21 +73,6 @@ def create_user( content=UserAlreadyExistsWithThisEmail(), ) - # Verify JWT token if OAuth-based registration - if ( - isinstance(user_to_create, Queries.CreateUserOauth) - and user_to_create.token != "" - ): - verification_result = verify_jwt_token(user_to_create.token) - if ( - verification_result is None - or verification_result.get("email") != user_to_create.email - ): - return JsonResponseWithStatus( - status_code=401, - content=InvalidOrExpiredJWTToken(), - ) - # Create the user in the database user = crud.create_user(db_session, user_to_create) diff --git a/src/backend/routers/user/delete.py b/src/backend/routers/user/delete.py index e227377..c0adf83 100644 --- a/src/backend/routers/user/delete.py +++ b/src/backend/routers/user/delete.py @@ -6,7 +6,11 @@ Features: - Cookie-based authentication using auth_token. -- Optional deletion of related data (sessions, queries). +- Always erases everything collected about the account (privacy.erasure), in + one transaction, before deleting the account itself. +- Refuses (409) accounts that own research studies or agent profiles, which + other people's study data depends on. +- Revokes the account's live tokens and clears its cookies. - Structured JSON responses with appropriate status codes. """ @@ -18,6 +22,7 @@ import database.crud as crud from App import App from backend.Responses import ( + DeleteUserBlockedError, DeleteUserDeleteResponse, DeleteUserError, ErrorResponse, @@ -25,10 +30,13 @@ JsonResponseWithStatus, UserNotFoundError, ) +from privacy import erasure # Initialize FastAPI router router = APIRouter() +_ACCOUNT_COOKIES = ("auth_token", "session_token", "project_token") + @router.delete( "", @@ -37,21 +45,25 @@ "200": {"model": DeleteUserDeleteResponse}, "401": {"model": InvalidOrExpiredAuthToken}, "404": {"model": UserNotFoundError}, + "409": {"model": DeleteUserBlockedError}, "422": {"model": ErrorResponse}, "429": {"model": ErrorResponse}, "500": {"model": DeleteUserError}, }, ) def delete_user( - delete_data: bool = Query(False, description="Delete user's associated data"), + delete_data: bool = Query( + False, + description="Deprecated and ignored: deleting an account always erases its data.", + ), auth_token: str = Cookie(""), app: App = Depends(App.get_instance), ) -> JsonResponseWithStatus: """ - Delete the authenticated user's account and optionally their associated data. + Delete the authenticated user's account together with all data collected about it. Args: - delete_data (bool): If True, removes all user-related data (default is False). + delete_data (bool): Ignored; kept so existing clients keep working. auth_token (str): Auth token provided in cookies to authenticate the user. app (App): Dependency-injected app instance with DB and Redis access. @@ -72,30 +84,23 @@ def delete_user( content=InvalidOrExpiredAuthToken(), ) - user_id = auth_info["user_id"] + user_id = uuid.UUID(auth_info["user_id"]) # Check if user exists in the database - found_user = crud.get_user_by_id(db_session, uuid.UUID(user_id)) - if not found_user: + if not crud.get_user_by_id(db_session, user_id): return JsonResponseWithStatus( status_code=404, content=UserNotFoundError(), ) - # Delete session/auth token data from Redis - redis_manager.delete("user_id", user_id, db_session) - - # Optionally delete all associated data - if delete_data: - logging.info(f"Performing full wipe-out for user ID: {user_id}") - crud.delete_user_full_wipe_out(db=db_session, user_id=uuid.UUID(user_id)) - - # Delete the user account itself - crud.delete_user_by_id(db=db_session, user_id=user_id) - - return JsonResponseWithStatus( - status_code=200, - content=DeleteUserDeleteResponse(), - ) + try: + erasure.delete_account(db_session, user_id) + except erasure.AccountDeletionBlocked as blocked: + db_session.rollback() + return JsonResponseWithStatus( + status_code=409, + content=DeleteUserBlockedError(message=str(blocked)), + ) + db_session.commit() except Exception as e: logging.error(f"Error processing user deletion request: {str(e)}") @@ -107,6 +112,22 @@ def delete_user( finally: db_session.close() + # The account is gone: none of its credentials may keep working. Best effort: + # a token that outlives a Redis failure names an account that no longer + # exists, and nothing is ever collected for an unknown account. + try: + redis_manager.revoke_user_tokens(str(user_id)) + except Exception as e: + logging.warning(f"Could not revoke the deleted account's tokens: {str(e)}") + + response = JsonResponseWithStatus( + status_code=200, + content=DeleteUserDeleteResponse(), + ) + for cookie in _ACCOUNT_COOKIES: + response.delete_cookie(cookie) + return response + def __init__(): """ diff --git a/src/backend/routers/user/privacy.py b/src/backend/routers/user/privacy.py new file mode 100644 index 0000000..a3bedf7 --- /dev/null +++ b/src/backend/routers/user/privacy.py @@ -0,0 +1,166 @@ +"""Self-service privacy controls for the signed-in account (GDPR Art. 7(3), 17). + +Mounted under ``/api/user/privacy``: + +* ``GET /`` — collection status, stored-data counts, deletion eligibility +* ``PUT /collection`` — opt out of data collection, or back in +* ``POST /erase`` — opt out and erase everything collected about the account + +The rules live in :mod:`privacy`; this router resolves the caller, owns the +transaction and shapes the response. Account deletion stays at +``DELETE /api/user/delete``. +""" + +from __future__ import annotations + +import logging +import uuid +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter, Depends +from pydantic import BaseModel, ConfigDict + +from App import App +from backend.routers.analytics.auth_utils import AuthenticatedUser, get_current_user +from database.db_schemas import Study +from privacy import collection, erasure +from research.participants import identity as participant_identity + +router = APIRouter() + + +class DataCollection(BaseModel): + enabled: bool + opted_out_at: Optional[datetime] = None + + +class ActiveStudy(BaseModel): + study_id: uuid.UUID + name: str + + +class StoredDataCounts(BaseModel): + queries: int + chats: int + agent_runs: int + study_enrollments: int + study_events: int + + +class AccountDeletion(BaseModel): + allowed: bool + blocked_reason: Optional[str] = None + + +class PrivacyStatus(BaseModel): + data_collection: DataCollection + active_study: Optional[ActiveStudy] = None + stored_data: StoredDataCounts + account_deletion: AccountDeletion + + +class CollectionUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + + enabled: bool + + +class ErasureResult(BaseModel): + erased: StoredDataCounts + status: PrivacyStatus + + +def _active_study(db, user_id: uuid.UUID) -> Optional[ActiveStudy]: + participant = participant_identity.get_participant_by_account(db, user_id) + if participant is None: + return None + enrollment = participant_identity.get_active_enrollment_for_participant( + db, participant.participant_id + ) + study = db.get(Study, enrollment.study_id) if enrollment is not None else None + return ActiveStudy(study_id=study.study_id, name=study.name) if study is not None else None + + +def _status(db, user_id: uuid.UUID) -> PrivacyStatus: + opted_out_at = collection.opted_out_at(db, user_id) + blocked_reason = erasure.account_deletion_blocker(db, user_id) + return PrivacyStatus( + data_collection=DataCollection(enabled=opted_out_at is None, opted_out_at=opted_out_at), + active_study=_active_study(db, user_id), + stored_data=StoredDataCounts(**erasure.stored_data_summary(db, user_id).as_dict()), + account_deletion=AccountDeletion( + allowed=blocked_reason is None, blocked_reason=blocked_reason + ), + ) + + +def _live_session_ids(app: App, user_id: uuid.UUID) -> list[uuid.UUID]: + """The account's live classic session, which the IDE keeps using after an erase.""" + user_info = app.get_redis_manager().get("user_token", str(user_id)) or {} + try: + return [uuid.UUID(str(user_info["session_token"]))] + except (KeyError, ValueError): + return [] + + +@router.get("", summary="Data-collection status and stored data of the signed-in account") +def get_privacy_status( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +) -> PrivacyStatus: + db = app.get_db_session() + try: + return _status(db, current_user.user_id) + finally: + db.close() + + +@router.put("/collection", summary="Opt out of data collection, or back in") +def update_data_collection( + payload: CollectionUpdate, + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +) -> PrivacyStatus: + db = app.get_db_session() + try: + if payload.enabled: + collection.opt_in(db, current_user.user_id) + else: + collection.opt_out(db, current_user.user_id) + db.commit() + return _status(db, current_user.user_id) + except Exception: + db.rollback() + raise + finally: + db.close() + + +@router.post("/erase", summary="Opt out and erase everything collected about the signed-in account") +def erase_my_data( + current_user: AuthenticatedUser = Depends(get_current_user), + app: App = Depends(App.get_instance), +) -> ErasureResult: + keep_session_ids = _live_session_ids(app, current_user.user_id) + db = app.get_db_session() + try: + erased = erasure.erase_collected_data( + db, current_user.user_id, keep_session_ids=keep_session_ids + ) + db.commit() + logging.info(f"[privacy] erased collected data: {erased.as_dict()}") + try: + # The live session's cached project context must not be written back. + app.get_redis_manager().mark_context_erased(str(current_user.user_id)) + except Exception as error: + logging.warning(f"[privacy] could not mark the live context erased: {error}") + return ErasureResult( + erased=StoredDataCounts(**erased.as_dict()), + status=_status(db, current_user.user_id), + ) + except Exception: + db.rollback() + raise + finally: + db.close() diff --git a/src/backend/routers/user/verify.py b/src/backend/routers/user/verify.py index a442e2a..385d0e7 100644 --- a/src/backend/routers/user/verify.py +++ b/src/backend/routers/user/verify.py @@ -18,7 +18,8 @@ VerifyUserError, VerifyUserGetHTMLResponse, ) -from celery_app.tasks.db_tasks import send_verification_email_task +from backend.email_utils import send_verification_email +from utils import create_uuid from Queries import UpdateUser # Initialize API router for user verification routes @@ -101,6 +102,7 @@ def check_verification( "422": {"model": ErrorResponse}, "429": {"model": ErrorResponse}, "500": {"model": ResendVerificationEmailError}, + "502": {"model": ResendVerificationEmailError}, }, tags=["User Verification"], ) @@ -144,10 +146,21 @@ def resend_verification_email( content=UserNotFoundError(), ) - # Enqueue a Celery task to send the verification email - send_verification_email_task.delay( - str(user.user_id), str(user.email), user.name + # Send in the request rather than through a background task: a queued + # task cannot report a delivery failure back to the person who clicked + # "Resend", so the page would claim success while nothing was sent. + verification_token = create_uuid() + redis_client.set( + "email_verification", + verification_token, + {"user_id": str(user.user_id)}, + force_reset_exp=True, ) + if not send_verification_email(str(user.email), user.name, verification_token): + logging.error(f"Verification email delivery failed for user {user.user_id}") + return JsonResponseWithStatus( + status_code=502, content=ResendVerificationEmailError() + ) # Return success response return JsonResponseWithStatus( diff --git a/src/backend/utils.py b/src/backend/utils.py index 64f0a80..520d415 100644 --- a/src/backend/utils.py +++ b/src/backend/utils.py @@ -5,8 +5,6 @@ from typing import Any, Dict, Union, get_args, get_origin from uuid import UUID -from google.auth.transport import requests -from google.oauth2 import id_token from polyfactory.factories.pydantic_factory import ModelFactory from polyfactory.field_meta import FieldMeta from pydantic import BaseModel, EmailStr, SecretStr @@ -199,20 +197,6 @@ def get_field_value(cls, field_meta: FieldMeta, *args, **kwargs): return _Factory.batch(size=n, **overrides) -def verify_jwt_token(token: str, provider: str = "google"): - """ - Verifies a JWT token using the provider's public key infrastructure. - Currently supports Google only. - """ - try: - if provider == "google": - id_info = id_token.verify_oauth2_token(token, requests.Request()) - email = id_info.get("email") - return {"email": email, "id_info": id_info} - except ValueError: - return None - - def recursive_json_loads(obj): """ Recursively attempts to JSON-decode any nested strings in the input structure. diff --git a/src/celery_app/celery_app.py b/src/celery_app/celery_app.py index 10a7fb2..31fb3ee 100644 --- a/src/celery_app/celery_app.py +++ b/src/celery_app/celery_app.py @@ -16,10 +16,14 @@ multiprocessing.set_start_method("spawn", force=True) -import torch # noqa: E402 - -# Ensure torch also uses spawn method for multiprocessing compatibility -torch.multiprocessing.set_start_method("spawn", force=True) +try: + import torch # noqa: E402 +except ImportError: + torch = None + +# Ensure torch also uses spawn method when the optional ML stack is installed. +if torch is not None: + torch.multiprocessing.set_start_method("spawn", force=True) from celery import Celery # noqa: E402 from dotenv import load_dotenv # noqa: E402 diff --git a/src/celery_app/tasks/chat_tasks.py b/src/celery_app/tasks/chat_tasks.py index 2354041..051dd9b 100644 --- a/src/celery_app/tasks/chat_tasks.py +++ b/src/celery_app/tasks/chat_tasks.py @@ -37,7 +37,8 @@ def chat_request_task( # Import at runtime to avoid circular import from backend.routers.chat.request import request_chat_completion - logging.log(logging.INFO, f"Processing chat completion request: {chat_request}") + # Never log the request itself: it carries the user's code and prompts. + logging.log(logging.INFO, f"Processing chat completion request for connection {connection_id}") app = App.get_instance() celery_broker = app.get_celery_broker() diff --git a/src/celery_app/tasks/db_tasks.py b/src/celery_app/tasks/db_tasks.py index 2565ad3..016cf0b 100644 --- a/src/celery_app/tasks/db_tasks.py +++ b/src/celery_app/tasks/db_tasks.py @@ -7,14 +7,57 @@ on errors and follow a consistent pattern for data validation and persistence. """ +import uuid + +from celery.exceptions import Ignore + import Queries from App import App from backend.email_utils import send_reset_password_email, send_verification_email from celery_app.celery_app import celery from database import crud +from privacy import collection from utils import create_uuid +def _stop_unless_collecting(db, query) -> None: + """ + Stop the chain if the account opted out after its request was accepted. + + The request's context and telemetry rows (stored by the chain's first step) + are discarded, and Ignore keeps the rest of the chain, including the + generations, from running. + """ + if collection.discard_if_opted_out( + db, + query.user_id, + context_id=query.context_id, + contextual_telemetry_id=query.contextual_telemetry_id, + behavioral_telemetry_id=query.behavioral_telemetry_id, + ): + db.commit() + raise Ignore() + + +def _query_owner_collecting(db, meta_query_id) -> bool: + """ + Whether a row that hangs off a stored query (generation, feedback) may be written. + + False when the query is gone (erased meanwhile) or its account opted out, so + the task skips instead of failing a foreign key or storing after an opt-out. + The opt-out is decided under the account-row lock; a query erased between + the read and the lock still makes the insert fail its foreign key. + """ + if meta_query_id is None: + return False + query = crud.get_meta_query_by_id(db, uuid.UUID(str(meta_query_id))) + return ( + query is not None + and query.user_id is not None + and collection.lock_collection_allowed(db, query.user_id) + ) + + @celery.task def add_context_task(context_data: dict, context_id: str = None): # type: ignore """ @@ -122,6 +165,7 @@ def add_completion_query_task(query_data: dict, query_id: str = None): # type: with App.get_instance().get_db_session() as db: try: query_query = Queries.CreateCompletionQuery(**query_data) + _stop_unless_collecting(db, query_query) crud.create_completion_query(db=db, query=query_query, id=query_id) except Exception: db.rollback() @@ -148,6 +192,7 @@ def add_chat_query_task(query_data: dict, query_id: str = None): # type: ignore with App.get_instance().get_db_session() as db: try: query_query = Queries.CreateChatQuery(**query_data) + _stop_unless_collecting(db, query_query) crud.create_chat_query(db=db, query=query_query, id=query_id) except Exception: db.rollback() @@ -174,7 +219,11 @@ def get_or_create_chat_task(chat_data: dict, chat_id: str = None): # type: igno with App.get_instance().get_db_session() as db: try: chat_query = Queries.CreateChat(**chat_data) - crud.create_chat(db=db, chat=chat_query, chat_id=chat_id) + # An opted-out account gets no chat row; the chain goes on to the + # chat query, which discards the request's other rows. The lock + # keeps an erase from slipping in before the chat row commits. + if collection.lock_collection_allowed(db, chat_query.user_id): + crud.create_chat(db=db, chat=chat_query, chat_id=chat_id) except Exception: db.rollback() raise @@ -200,7 +249,8 @@ def add_generation_task(generation_data: dict, generation_id: str = None): # ty with App.get_instance().get_db_session() as db: try: generation_query = Queries.CreateGeneration(**generation_data) - crud.create_generation(db=db, generation=generation_query, id=generation_id) + if _query_owner_collecting(db, generation_id): + crud.create_generation(db=db, generation=generation_query, id=generation_id) except Exception: db.rollback() raise @@ -227,9 +277,10 @@ def update_generation_task(query_id: str, model_id: int, generation_data: dict): with App.get_instance().get_db_session() as db: try: generation_query = Queries.UpdateGeneration(**generation_data) - crud.update_generation( - db=db, query_id=query_id, model_id=model_id, generation=generation_query - ) + if _query_owner_collecting(db, query_id): + crud.update_generation( + db=db, query_id=query_id, model_id=model_id, generation=generation_query + ) except Exception: db.rollback() raise @@ -254,7 +305,8 @@ def add_ground_truth_task(ground_truth_data: dict): with App.get_instance().get_db_session() as db: try: ground_truth_query = Queries.CreateGroundTruth(**ground_truth_data) - crud.create_ground_truth(db=db, ground_truth=ground_truth_query) + if _query_owner_collecting(db, ground_truth_query.completion_query_id): + crud.create_ground_truth(db=db, ground_truth=ground_truth_query) except Exception: db.rollback() raise @@ -288,8 +340,10 @@ def send_verification_email_task(user_id: str, user_email: str, user_name: str): force_reset_exp=True, ) - # Send the verification email with the generated token - send_verification_email(user_email, user_name, verification_token) + # Propagate delivery failures so Celery marks the job as failed instead of + # reporting a false success to operators. + if not send_verification_email(user_email, user_name, verification_token): + raise RuntimeError(f"Verification email delivery failed for user {user_id}") @celery.task diff --git a/src/celery_app/tasks/llm_tasks.py b/src/celery_app/tasks/llm_tasks.py index 16d81c5..4fa760f 100644 --- a/src/celery_app/tasks/llm_tasks.py +++ b/src/celery_app/tasks/llm_tasks.py @@ -36,7 +36,8 @@ def completion_request_task( # Import at runtime to avoid circular import from backend.routers.completion.request import request_completion - logging.log(logging.INFO, f"Processing completion request: {completion_request}") + # Never log the request itself: it carries the user's code. + logging.log(logging.INFO, f"Processing completion request for connection {connection_id}") app = App.get_instance() celery_broker = app.get_celery_broker() diff --git a/src/code4me2_agent/__init__.py b/src/code4me2_agent/__init__.py new file mode 100644 index 0000000..4c7d33d --- /dev/null +++ b/src/code4me2_agent/__init__.py @@ -0,0 +1,16 @@ +from code4me2_agent.adapters import AdapterResult, AgentAdapter, create_agent_adapter +from code4me2_agent.command_tools import WorkspaceCommandTools +from code4me2_agent.config import AgentConfig +from code4me2_agent.echo import EchoAgentCore, EchoPromptResult +from code4me2_agent.file_tools import WorkspaceFileTools + +__all__ = [ + "AdapterResult", + "AgentAdapter", + "AgentConfig", + "EchoAgentCore", + "EchoPromptResult", + "WorkspaceFileTools", + "WorkspaceCommandTools", + "create_agent_adapter", +] diff --git a/src/code4me2_agent/_build_version.py b/src/code4me2_agent/_build_version.py new file mode 100644 index 0000000..791c230 --- /dev/null +++ b/src/code4me2_agent/_build_version.py @@ -0,0 +1,3 @@ +"""Generated by packaging/stamp_runtime_version.py.""" + +RUNTIME_VERSION = "0.1.0-gt-telemetry.20260923" diff --git a/src/code4me2_agent/acp_runtime.py b/src/code4me2_agent/acp_runtime.py new file mode 100644 index 0000000..84500e6 --- /dev/null +++ b/src/code4me2_agent/acp_runtime.py @@ -0,0 +1,1694 @@ +from __future__ import annotations + +import asyncio +import json +import logging +import os +import platform +import re +from dataclasses import dataclass, field, replace +from pathlib import Path +from threading import Event +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +from code4me2_agent import slash_commands +from code4me2_agent.acp_updates import ( + REVISE_INSTRUCTIONS_MAX_CHARS, + REVISE_OPTION_ID, + AcpUpdateBuilder, +) +from code4me2_agent.acp_utils import capability_value +from code4me2_agent.async_bridge import EventLoopAsyncRunner, OperationCancelled +from code4me2_agent.command_tools import build_acp_command_backend +from code4me2_agent.echo import EchoAgentCore +from code4me2_agent.events import ( + ApprovalDecision, + AssistantTextEvent, + PlanEvent, + ThoughtEvent, + UsageEvent, +) +from code4me2_agent.tool_catalog import approval_kind, tool_kind +from code4me2_agent.file_tools import build_acp_file_system_backend +from code4me2_agent.mcp_tools import ( + StdioMcpToolBroker, + mcp_server_transport, + serialize_mcp_servers, +) +from code4me2_agent.runtime_auth import ( + AcpAuthorizationFailure, + AcpBackendAuthorization, + ManagedBridgeAuthorization, +) + +logger = logging.getLogger(__name__) + +# The runtime currently implements ACP protocol version 1. +SUPPORTED_PROTOCOL_VERSIONS: tuple[int, ...] = (1,) + +# Python SDK 0.12.1 still guards the now-stable session/resume and session/close +# routes with this SDK-wide switch. Unsupported guarded routes are safe because +# the concrete agent below deliberately does not inherit the SDK Protocol stubs. +_ENABLE_STABLE_SESSION_ROUTE_WORKAROUND = True + + +if TYPE_CHECKING: + from acp.interfaces import Client + + from code4me2_agent.config import AgentConfig, CommandConfig + from code4me2_agent.events import ToolCallEvent + from code4me2_agent.hunks import RevisionOffer + from code4me2_agent.telemetry import AgentTelemetryRecorder + + +# Attached file contents inlined into a prompt, in characters (about a quarter +# of the context budget, never more than this cap). +_MAX_ATTACHMENT_CHARS = 60_000 +_MIN_ATTACHMENT_CHARS = 4_000 + + +async def _prompt_text_async( + prompt: list[Any], + *, + workspace_root: Path | None = None, + attachment_budget_chars: int = 32_000, +) -> str: + parts: list[str] = [] + budget = [max(_MIN_ATTACHMENT_CHARS, min(_MAX_ATTACHMENT_CHARS, int(attachment_budget_chars)))] + for block in prompt: + if isinstance(block, dict): + block_type = block.get("type", "") + text = block.get("text", "") + else: + block_type = getattr(block, "type", "") + text = getattr(block, "text", "") + if block_type == "resource_link": + text = await _resource_link_text_async(block) + elif block_type == "resource": + text = _embedded_resource_text(block, workspace_root=workspace_root, budget=budget) + if text: + parts.append(text) + return "\n".join(parts) + + +def _embedded_resource_text(block: object, *, workspace_root: Path | None, budget: list[int]) -> str: + """Inline an ``@file``/selection attachment so the model needs no read call.""" + resource = capability_value(block, "resource") + uri = str(capability_value(resource, "uri") or "").strip() + mime_type = capability_value(resource, "mimeType") or capability_value(resource, "mime_type") + label = _attachment_label(uri, workspace_root) + text = capability_value(resource, "text") + if not isinstance(text, str): + detail = f", {mime_type}" if mime_type else "" + return f"[Attached binary resource: {label}{detail}; its content is not shown.]" + remaining = budget[0] + shown = text[:remaining] if remaining > 0 else "" + budget[0] = max(0, remaining - len(shown)) + header = f'" + body = shown + if len(shown) < len(text): + body += ( + f"\n[attachment truncated: showing {len(shown)} of {len(text)} characters; " + "read the file for the rest]" + ) + return f"{header}\n{body}\n" + + +def _attachment_label(uri: str, workspace_root: Path | None) -> str: + if uri.startswith("file://"): + from urllib.parse import unquote, urlparse + from urllib.request import url2pathname + + raw_path = urlparse(uri).path + try: + # "/C:/Users/..." -> "C:\Users\..." on Windows; plain unquote on POSIX. + path = Path(url2pathname(raw_path)) + except OSError: # older Windows Pythons reject some colon placements + path = Path(unquote(raw_path)) + if workspace_root is not None: + try: + return path.resolve().relative_to(workspace_root.resolve()).as_posix() + except (OSError, ValueError): + pass + return path.as_posix() + return uri or "attachment" + + +async def _resource_link_text_async(block: object) -> str: + uri = str(capability_value(block, "uri") or "").strip() + name = str(capability_value(block, "name") or uri or "resource").strip() + details = [] + mime_type = capability_value(block, "mimeType") or capability_value( + block, "mime_type" + ) + description = capability_value(block, "description") + if mime_type: + details.append(f"mimeType={mime_type}") + if description: + details.append(f"description={description}") + suffix = f"; {'; '.join(details)}" if details else "" + resource_link = f"[Resource link: {name} <{uri}>{suffix}]" + return resource_link + + +# The parts _prompt_text_async adds for what the client attached to a prompt +# (each on lines of its own), with the line break joining it to the previous part. +_ATTACHMENT_PARTS = re.compile( + r"\n?(?:^\[Resource link: [^\n]*\]$" + r"|^\[Attached binary resource: [^\n]*\]$" + r"|^\n.*?\n$)", + re.MULTILINE | re.DOTALL, +) + + +def _typed_prompt_text(model_text: str) -> str: + """What the participant typed, from a prompt's model text: a replayed chat + shows their words, not the copies of attached files the model was given.""" + typed = _ATTACHMENT_PARTS.sub("", model_text).strip() + return typed or model_text + +_ACP_STOP_REASONS = frozenset( + {"end_turn", "max_tokens", "max_turn_requests", "refusal", "cancelled"} +) +_USAGE_UPDATES_ENABLED = os.getenv("CODE4ME_ACP_USAGE_UPDATES", "1").strip().lower() not in { + "0", + "false", + "no", + "off", +} + + +def _acp_stop_reason(adapter_stop_reason: str) -> str: + """Map adapter outcomes onto the ACP StopReason vocabulary. + + Failures (``error``, ``provider_exhausted``) have already been shown to the + user as a final agent message, so the turn ended normally from the client's + point of view; the cause travels in ``PromptResponse._meta``. ``refusal`` is + reserved for a model content-filter refusal. + """ + if adapter_stop_reason == "max_iterations": + return "max_turn_requests" + if adapter_stop_reason in _ACP_STOP_REASONS: + return adapter_stop_reason + return "end_turn" + + +@dataclass +class AgentSession: + session_id: str + core: EchoAgentCore + mcp_tools: StdioMcpToolBroker | None = None + cancel_event: Event = field(default_factory=Event) + prompt_lock: asyncio.Lock = field(default_factory=asyncio.Lock) + active_prompt_task: asyncio.Task[Any] | None = None + authorization: Any | None = None + title: str | None = None + + +class AcpSessionEventSink: + def __init__( + self, + *, + conn: Client, + session_id: str, + updates: AcpUpdateBuilder, + telemetry: object, + async_runner: EventLoopAsyncRunner, + cancel_event: Event | None = None, + elicitation_form: bool = False, + ) -> None: + self._conn = conn + self._session_id = session_id + self._updates = updates + self._telemetry = telemetry + self._async_runner = async_runner + self._cancel_event = cancel_event + # The client advertised elicitation.form: approvals may offer "Revise…". + self._elicitation_form = elicitation_form + self._session_approved_kinds: set[str] = set() + self._tool_content: dict[str, list[Any] | None] = {} + + def _run_blocking(self, awaitable: Any) -> Any: + if self._cancel_event is None: + return self._async_runner.run(awaitable) + return self._async_runner.run(awaitable, cancel_event=self._cancel_event) + + def assistant_text(self, event: AssistantTextEvent) -> None: + update = self._updates.agent_message( + event.text, + message_id=event.message_id, + metadata={ + "code4me2": { + "phase": "final" if event.final else "progress", + "iteration": event.iteration, + } + }, + ) + self._send_session_update( + update, + run_id=event.run_id, + request_id=event.request_id, + update_type="agent_message_chunk", + ) + + def plan(self, event: PlanEvent) -> None: + update = self._updates.agent_plan(list(event.entries)) + self._send_session_update( + update, + run_id=event.run_id, + request_id=event.request_id, + update_type="plan", + extra={"tool_call_id": event.tool_call_id}, + ) + + def usage(self, event: UsageEvent) -> None: + if not _USAGE_UPDATES_ENABLED: + return + update = self._updates.usage_update( + used=event.prompt_tokens + event.completion_tokens, + size=event.context_budget_tokens, + metadata={ + "code4me2": { + "iteration": event.iteration, + "model": event.model, + "promptTokens": event.prompt_tokens, + "completionTokens": event.completion_tokens, + "totalTokens": event.total_tokens, + "turnTotalTokens": event.turn_total_tokens, + } + }, + ) + self._send_session_update( + update, + run_id=event.run_id, + request_id=event.request_id, + update_type="usage_update", + ) + + def tool_call(self, event: ToolCallEvent) -> None: + content = self._tool_call_content(event) + if event.phase == "progress": + # Streamed output of a running command: replace the card body only. + update = self._updates.update_tool_call( + tool_call_id=event.tool_call_id, + status="in_progress", + content=content, + ) + self._send_session_update( + update, + run_id=event.run_id, + request_id=event.request_id, + update_type="tool_call_update", + extra={ + "tool_call_id": event.tool_call_id, + "tool_name": event.tool_name, + "phase": event.phase, + }, + ) + return + if event.phase == "started": + self._tool_content[event.tool_call_id] = content + update = self._updates.start_tool_call( + tool_call_id=event.tool_call_id, + title=event.title, + kind=event.kind, + status=event.status, + path=event.path, + locations=event.locations, + content=content, + raw_input=event.raw_input, + ) + else: + update = self._updates.update_tool_call( + tool_call_id=event.tool_call_id, + title=event.title, + kind=event.kind, + status=event.status, + content=content, + raw_output=event.raw_output, + ) + self._send_session_update( + update, + run_id=event.run_id, + request_id=event.request_id, + update_type=str(getattr(update, "session_update", "tool_call")), + extra={ + "tool_call_id": event.tool_call_id, + "tool_name": event.tool_name, + "phase": event.phase, + }, + ) + if event.phase != "started": + self._tool_content.pop(event.tool_call_id, None) + + def _tool_call_content(self, event: ToolCallEvent) -> list[Any] | None: + content: list[Any] = [] + if event.path and event.diff_new_text is not None: + content.append( + self._updates.diff_tool_content( + event.path, + old_text=event.diff_old_text, + new_text=event.diff_new_text, + ) + ) + for path, old_text, new_text in getattr(event, "extra_diffs", ()) or (): + content.append( + self._updates.diff_tool_content(path, old_text=old_text, new_text=new_text) + ) + if event.content_text: + content.append(self._updates.text_tool_content(event.content_text)) + return content or None + + def thought(self, event: object) -> None: + phase = getattr(event, "phase", None) + duration_ms = getattr(event, "duration_ms", None) + timing_metadata = None + if phase is not None: + phase_metadata: dict[str, object] = {"phase": phase} + if duration_ms is not None: + phase_metadata["durationMs"] = duration_ms + timing_metadata = {"code4me2": phase_metadata} + update = self._updates.agent_thought( + getattr(event, "text", ""), + metadata=timing_metadata, + ) + self._send_session_update( + update, + run_id=getattr(event, "run_id", None), + request_id=getattr(event, "request_id", None), + update_type="agent_thought", + ) + + def request_approval( + self, + tool_call: object, + arguments: dict[str, Any], + *, + revise: RevisionOffer | None = None, + ) -> ApprovalDecision: + """Synchronously bridge a worker-thread tool decision to ACP/JetBrains. + + With ``revise`` and a client that renders ACP forms, a fourth option + "Revise…" opens a form for the parts to keep and what should change. + """ + name = str(getattr(tool_call, "name", "tool")) + tool_call_id = str(getattr(tool_call, "tool_call_id", "")) + metadata = approval_kind(name) + if metadata in self._session_approved_kinds: + # Answered by the user's earlier "allow for this session": no one is + # asked, so telemetry must not report a new decision. + return ApprovalDecision("accepted", "session_cached") + offer = revise if self._elicitation_form else None + summary = _approval_summary(name, arguments) + permission = self._updates.permission_request( + session_id=self._session_id, + tool_call_id=tool_call_id, + title=summary, + kind=tool_kind(name), + summary=summary, + raw_input=_approval_raw_input(name, arguments), + session_option_name=_session_option_name(metadata), + content=self._tool_content.get(tool_call_id), + revise=offer is not None, + ) + try: + response = self._run_blocking( + self._conn.request_permission( + session_id=permission.session_id, + tool_call=permission.tool_call, + options=permission.options, + ) + ) + except OperationCancelled: + return ApprovalDecision("cancelled") + except Exception: # noqa: BLE001 + logger.exception("ACP permission request failed for tool %s", name) + return ApprovalDecision("unavailable") + outcome = getattr(response, "outcome", None) + if isinstance(response, dict): + outcome = response.get("outcome") + if isinstance(outcome, dict): + selected = outcome.get("outcome") == "selected" + option_id = outcome.get("optionId", outcome.get("option_id")) + else: + selected = getattr(outcome, "outcome", None) == "selected" + option_id = getattr(outcome, "option_id", None) + if not selected: + return ApprovalDecision("cancelled") + if offer is not None and option_id == REVISE_OPTION_ID: + return self._request_revision(tool_call_id, offer) + scope = { + "allow_once": "once", + "allow_session": "session", + }.get(option_id) + if scope == "session": + self._session_approved_kinds.add(metadata) + return ApprovalDecision("accepted", scope) if scope else ApprovalDecision("rejected") + + def _request_revision(self, tool_call_id: str, offer: RevisionOffer) -> ApprovalDecision: + """Ask with an ACP form (``elicitation/create``) what to keep and what should change. + + An accepted form is decision ``revised``; a declined or dismissed one, + or a form that could not be shown, is a rejection; a turn cancelled + while it is open is ``cancelled``. + """ + hunks = offer.selectable_hunks + mode = self._updates.revision_form( + session_id=self._session_id, + tool_call_id=tool_call_id, + hunk_labels=[hunk.label for hunk in hunks], + ) + message = ( + "Which parts should be kept, and what should change?" + if hunks + else "What should change?" + ) + try: + response = self._run_blocking( + self._conn.create_elicitation(message=message, mode=mode) + ) + except OperationCancelled: + return ApprovalDecision("cancelled") + except Exception: # noqa: BLE001 + logger.exception("ACP revision form failed for tool call %s", tool_call_id) + return ApprovalDecision("rejected", elicitation_action="error") + action = capability_value(response, "action") + if action == "accept": + content = capability_value(response, "content") + instructions = capability_value(content, "instructions") + return ApprovalDecision( + "revised", + elicitation_action="accept", + kept_hunks=_kept_hunk_indexes(capability_value(content, "keep"), len(hunks)), + instructions=( + instructions.strip()[:REVISE_INSTRUCTIONS_MAX_CHARS] + if isinstance(instructions, str) + else "" + ), + ) + if action in ("decline", "cancel"): + return ApprovalDecision("rejected", elicitation_action=str(action)) + # An action this runtime does not know counts as a form that failed. + return ApprovalDecision("rejected", elicitation_action="error") + + def _send_session_update( + self, + update: object, + *, + run_id: str | None, + request_id: str | None, + update_type: str, + extra: dict[str, Any] | None = None, + ) -> None: + try: + self._run_blocking( + self._conn.session_update( + session_id=self._session_id, + update=update, + source="code4me2_agent", + ) + ) + except OperationCancelled: + return + except Exception as exc: # noqa: BLE001 + record = getattr(self._telemetry, "record", None) + if callable(record): + record( + event_type="agent.acp.update_failed", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "update_type": update_type, + "error_message": str(exc), + **(extra or {}), + }, + ) + + +def _record_acp_runtime_event( + telemetry: AgentTelemetryRecorder, + *, + event_type: str, + session_id: str, + request_id: str, + payload: dict[str, Any], +) -> None: + telemetry.record( + event_type=event_type, + run_id=f"acp-runtime-{request_id}", + request_id=request_id, + parent_event_id=None, + payload={"session_id": session_id, **payload}, + ) + + +def _approval_raw_input(name: str, arguments: dict[str, Any]) -> dict[str, Any] | None: + if name == "run_command": + argv = arguments.get("argv") + raw: dict[str, Any] = { + "argv": [str(arg) for arg in argv] if isinstance(argv, (list, tuple)) else [], + "cwd": str(arguments.get("cwd", ".")), + } + if arguments.get("timeout_seconds") is not None: + raw["timeout_seconds"] = arguments["timeout_seconds"] + return raw + if name in {"create_file", "write_file", "replace_text", "delete_file"}: + return {"path": str(arguments.get("path", ""))} + if name == "edit_file": + return { + "path": str(arguments.get("path", "")), + "edit_count": len(arguments.get("edits") or []), + } + if name == "move_file": + return { + "source_path": str(arguments.get("source_path", "")), + "destination_path": str(arguments.get("destination_path", "")), + "overwrite": bool(arguments.get("overwrite", False)), + } + return None + + +def _approval_summary(name: str, arguments: dict[str, Any]) -> str: + if name == "run_command": + argv = arguments.get("argv") + command = " ".join(str(arg) for arg in argv) if isinstance(argv, (list, tuple)) else "command" + timeout = arguments.get("timeout_seconds") + suffix = f" (timeout {int(float(timeout))}s)" if isinstance(timeout, (int, float)) else "" + return f"Run command: {command}{suffix}" + if name == "move_file": + source = str(arguments.get("source_path", "")).strip() + destination = str(arguments.get("destination_path", "")).strip() + return f"Move file: {source} → {destination}" + path = str(arguments.get("path", "")).strip() + if name == "edit_file": + edit_count = len(arguments.get("edits") or []) + return f"Edit file: {path} ({edit_count} edit{'s' if edit_count != 1 else ''})" + if name.startswith("mcp__"): + parts = name.split("__", 2) + return f"Call {parts[1]}: {parts[2]}" if len(parts) == 3 else f"Run tool: {name}" + labels = { + "create_file": "Create file", + "write_file": "Write file", + "replace_text": "Edit file", + "delete_file": "Delete file", + } + return f"{labels.get(name, f'Run tool: {name}')}{f': {path}' if path else ''}" + + +def _session_option_name(kind: str) -> str: + return { + "edit": "Allow edits for session", + "execute": "Allow commands for session", + "other": "Allow MCP tools for session", + }[kind] + + +def _supports_form_elicitation(client_capabilities: object | None) -> bool: + """Whether ``initialize`` advertised ``elicitation.form`` (``{}`` means yes).""" + form = capability_value(capability_value(client_capabilities, "elicitation"), "form") + return form is not None and form is not False + + +def _kept_hunk_indexes(value: object, hunk_count: int) -> tuple[int, ...]: + """The listed hunks a "Revise…" form kept, from its ``keep`` option values.""" + if not isinstance(value, (list, tuple)): + return () + kept: set[int] = set() + for item in value: + text = str(item).strip() + if text.isdecimal() and int(text) < hunk_count: + kept.add(int(text)) + return tuple(sorted(kept)) + + +def _normalize_capabilities(value: object | None) -> object | None: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, dict): + return {str(key): _normalize_capabilities(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_normalize_capabilities(item) for item in value] + if hasattr(value, "model_dump"): + dump = getattr(value, "model_dump") + if callable(dump): + return _normalize_capabilities(dump(by_alias=True, exclude_none=True)) + if hasattr(value, "__dict__"): + return { + str(key): _normalize_capabilities(item) + for key, item in vars(value).items() + if not str(key).startswith("_") + } + return repr(value) + + +def _prompt_backend_state(session: AgentSession) -> dict[str, Any]: + file_backend = getattr(session.core.file_tools, "_acp_backend", None) + command_backend = getattr(session.core.command_tools, "_acp_backend", None) + return { + "workspace_root": session.core._config.workspace_root.as_posix(), + "file_backend": "acp" if file_backend is not None else "local", + "file_backend_session_id": getattr(file_backend, "session_id", None), + "read_text_file_enabled": bool( + getattr(file_backend, "read_text_file_enabled", False) + ), + "write_text_file_enabled": bool( + getattr(file_backend, "write_text_file_enabled", False) + ), + "command_backend": "acp" if command_backend is not None else "local", + "terminal_enabled": bool(getattr(command_backend, "terminal_enabled", False)), + } + + +def _persisted_memory_messages(payload: dict) -> list[dict[str, Any]]: + messages = payload.get("messages", []) + if not isinstance(messages, list): + return [] + return [message for message in messages if isinstance(message, dict)] + + +def _is_supported_stdio_mcp_server(server: object) -> bool: + """Whether an ACP MCP entry is well formed for its transport, without connecting. + + stdio needs name/command/args/env, HTTP and SSE need name/url/headers (the + 0.12.1 shapes). Malformed entries are rejected up front; a well-formed + entry that later fails to start (missing binary, unreachable URL) only + costs its own tools, reported by the broker. + """ + transport = mcp_server_transport(server) + if transport == "stdio": + keys: tuple[str, ...] = ("name", "command", "args", "env") + elif transport in ("http", "sse"): + keys = ("name", "url", "headers") + else: + return False + return all(capability_value(server, key) is not None for key in keys) + + +def _should_record_runtime_context( + *, + client_capabilities: object | None, + session_lookup: str = "existing", +) -> bool: + return client_capabilities is not None or session_lookup != "existing" + + +def create_acp_agent( + config: AgentConfig, + *, + authorization: Any | None = None, +) -> Any: + try: + from acp import ( + AuthenticateResponse, + InitializeResponse, + LoadSessionResponse, + NewSessionResponse, + PromptResponse, + RequestError, + ) + from acp.schema import ( + AgentCapabilities, + AuthMethodAgent, + Implementation, + McpCapabilities, + PromptCapabilities, + ResumeSessionResponse, + SessionCapabilities, + SessionCloseCapabilities, + SessionResumeCapabilities, + Usage, + ) + except ImportError as exc: + raise SystemExit( + "The ACP SDK is required to run stdio mode. " + "Install this package from source with agent-client-protocol available." + ) from exc + + class Code4MeEchoAgent: + _conn: Client + _client_capabilities: object | None + _updates: AcpUpdateBuilder + _bootstrap_core: EchoAgentCore + _sessions: dict[str, AgentSession] + _authorization: Any + _authorized_workspace: Path | None + + def on_connect(self, conn: Client) -> None: + self._conn = conn + self._client_capabilities = None + self._updates = AcpUpdateBuilder() + self._sessions = {} + self._authorization = ( + authorization + or (ManagedBridgeAuthorization() if config.managed_mode else None) + or AcpBackendAuthorization.from_environment( + workspace_root=config.workspace_root, + ) + ) + self._authorized_workspace = None + self._current_config = config + self._background_tasks: set[asyncio.Task[Any]] = set() + self._bootstrap_core = EchoAgentCore(config) + self.file_tools = self._bootstrap_core.file_tools + self.command_tools = self._bootstrap_core.command_tools + + def _build_session( + self, + *, + session_config: AgentConfig, + session_id: str, + mcp_tools: StdioMcpToolBroker | None, + session_authorization: Any | None = None, + ) -> AgentSession: + async_runner = EventLoopAsyncRunner(asyncio.get_running_loop()) + cancel_event = Event() + event_sink = AcpSessionEventSink( + conn=self._conn, + session_id=session_id, + updates=self._updates, + telemetry=None, + async_runner=async_runner, + cancel_event=cancel_event, + elicitation_form=_supports_form_elicitation(self._client_capabilities), + ) + core = EchoAgentCore(session_config, event_sink=event_sink) + event_sink._telemetry = core._telemetry + core.rebuild_tools( + acp_file_backend=build_acp_file_system_backend( + client=self._conn, + session_id=session_id, + client_capabilities=self._client_capabilities, + async_runner=async_runner, + ), + acp_command_backend=build_acp_command_backend( + client=self._conn, + session_id=session_id, + client_capabilities=self._client_capabilities, + async_runner=async_runner, + ), + mcp_tools=mcp_tools, + ) + return AgentSession( + session_id=session_id, + core=core, + mcp_tools=mcp_tools, + cancel_event=cancel_event, + authorization=session_authorization or self._authorization, + ) + + def _load_persisted_memory(self, session: AgentSession) -> None: + request_json = getattr(session.authorization, "authorized_json_request", None) + if not callable(request_json): + return + try: + payload = request_json( + "GET", + f"/api/agent/memory/{session.session_id}", + None, + ) + except AcpAuthorizationFailure as exc: + logger.info("Could not load persisted ACP memory: %s", exc) + return + messages = _persisted_memory_messages(payload) + if messages: + session.core.load_session_memory(messages) + + def _save_persisted_memory(self, session: AgentSession) -> None: + snapshot = session.core.session_memory_snapshot() + if snapshot is None: + return + request_json = getattr(session.authorization, "authorized_json_request", None) + if not callable(request_json): + return + try: + request_json( + "PUT", + f"/api/agent/memory/{session.session_id}", + {"messages": snapshot}, + ) + except AcpAuthorizationFailure as exc: + logger.info("Could not save persisted ACP memory: %s", exc) + + def _session_config( + self, *, workspace_root: Path, session_id: str, session_authorization: Any + ) -> AgentConfig: + # Build from self._current_config, NOT the module-level `config` + # closure: _apply_server_config writes the assigned agent profile + # into _current_config, and reading the original file config here + # would silently drop the server's model/provider/tool overrides + # for every session created after authentication — i.e. the A/B + # assignment would appear to work (it's logged) but never actually + # take effect on a new session. + base_config = self._current_config + telemetry_headers = session_authorization.telemetry_headers() + provider_auth_headers = ( + telemetry_headers + if base_config.adapter.provider.kind == "code4me_backend" + else base_config.adapter.provider.auth_headers + ) + upload_config = base_config.upload + backend_url = getattr(session_authorization, "backend_url", None) + if backend_url and upload_config.ingest_url is None: + # Self-reported telemetry converges on the same agent_event + # table the inference relay writes to (merge decision 2). + upload_config = replace( + upload_config, + enabled=True, + ingest_url=f"{str(backend_url).rstrip('/')}/api/agent/events/ingest", + ) + return replace( + base_config, + workspace_root=workspace_root, + session_id=session_id, + upload=replace( + upload_config, + auth_headers=telemetry_headers, + auth_headers_provider=session_authorization.telemetry_headers, + ), + adapter=replace( + base_config.adapter, + provider=replace( + base_config.adapter.provider, + auth_headers=provider_auth_headers, + ), + ), + managed_request=( + session_authorization.managed_inference + if base_config.managed_mode + else base_config.managed_request + ), + ) + + async def _create_or_resume_session( + self, + *, + cwd: str, + session_id: str, + event_type: str, + mcp_servers: list[Any] | None, + ) -> AgentSession: + workspace_root = _resolve_session_cwd(cwd) + session_authorization = self._authorization + if config.managed_mode: + session_authorization = ManagedBridgeAuthorization() + try: + await asyncio.to_thread(session_authorization.prepare_workspace, workspace_root) + scope = await asyncio.to_thread(session_authorization.authenticate) + except AcpAuthorizationFailure: + raise RequestError.auth_required({"reason": "authorization_rejected"}) from None + if _resolve_session_cwd(scope.workspace) != workspace_root: + raise RequestError.auth_required({"reason": "workspace_not_authorized"}) + else: + await self._require_authenticated() + if workspace_root != self._authorized_workspace: + raise RequestError.auth_required({"reason": "workspace_not_authorized"}) + base_config = self._current_config + server_config = session_authorization.server_agent_config + if config.managed_mode: + # Managed runs never fall back to local defaults: a missing + # profile, an unsupported runtime, or a malformed policy must + # fail before study work starts. + if server_config is None or not server_config.has_overrides: + raise RequestError.invalid_params( + {"reason": "managed_config_unavailable"} + ) + if ( + server_config.framework_version + and server_config.framework_version + not in {"code4me2-agent", "code4me-agent"} + ): + raise RequestError.invalid_params( + {"reason": "unsupported_managed_runtime", "runtime": server_config.framework_version} + ) + base_config = replace(base_config, workspace_root=workspace_root).with_server_overrides( + server_config, + backend_url=getattr(session_authorization, "backend_url", None), + ) + elif server_config is not None: + base_config = replace(base_config, workspace_root=workspace_root).with_server_overrides( + server_config, + backend_url=getattr(session_authorization, "backend_url", None), + ) + previous_config = self._current_config + self._current_config = base_config + session_config = self._session_config( + workspace_root=workspace_root, + session_id=session_id, + session_authorization=session_authorization, + ) + self._current_config = previous_config + requested_mcp_servers = list(mcp_servers or []) + mcp_allowed_by_policy = ( + not config.managed_mode + or ( + base_config.approval_policy != "suggestion_only" + and base_config.allowed_tools is not None + and any(name.startswith("mcp__") for name in base_config.allowed_tools) + ) + ) + enabled_mcp_servers = requested_mcp_servers if mcp_allowed_by_policy else [] + if requested_mcp_servers and not mcp_allowed_by_policy: + logger.info( + "Assigned policy disabled %d client-supplied MCP server(s) before launch.", + len(requested_mcp_servers), + ) + mcp_failures: list[dict[str, str]] = [] + try: + mcp_tools = await asyncio.to_thread( + StdioMcpToolBroker.open, + enabled_mcp_servers, + cwd=workspace_root, + ) + except (RuntimeError, TimeoutError, ValueError) as exc: + mcp_failures = list(getattr(exc, "failures", None) or []) + # Best effort: a failing tool sidecar (e.g. the IDE-bundled MCP + # server binary refusing a second instance) must not take down + # the whole chat. The session keeps its native file/terminal + # tools; malformed entries are still rejected up front. + logger.warning( + "ACP session continuing without MCP tools after broker failure %s: %s", + serialize_mcp_servers(mcp_servers), + exc, + ) + mcp_tools = None + try: + session = self._build_session( + session_config=session_config, + session_id=session_id, + mcp_tools=mcp_tools, + session_authorization=session_authorization, + ) + except BaseException: + if mcp_tools is not None: + await asyncio.to_thread(mcp_tools.close) + raise + await asyncio.to_thread(self._load_persisted_memory, session) + previous_session = self._sessions.get(session_id) + if previous_session is not None: + await self._stop_session(previous_session) + self._sessions[session_id] = session + self.file_tools = session.core.file_tools + self.command_tools = session.core.command_tools + if _should_record_runtime_context( + client_capabilities=self._client_capabilities + ): + _record_acp_runtime_event( + session.core._telemetry, + event_type=event_type, + session_id=session_id, + request_id=uuid4().hex, + payload={ + "cwd": session_config.workspace_root.as_posix(), + "runtime_os": platform.system(), + "runtime_architecture": platform.machine(), + "blocked_commands": list( + session_config.commands.blocked_commands + ), + "client_capabilities": _normalize_capabilities( + self._client_capabilities + ), + "mcp_servers_requested": serialize_mcp_servers(requested_mcp_servers), + "mcp_servers_policy_enabled": bool(enabled_mcp_servers), + "mcp_backend": "broker" + if session.mcp_tools is not None + else "none", + "mcp_servers_connected": list( + getattr(session.mcp_tools, "connected_servers", None) or [] + ), + "mcp_server_failures": mcp_failures + or list(getattr(session.mcp_tools, "failures", None) or []), + **_prompt_backend_state(session), + }, + ) + return session + + @staticmethod + async def _stop_active_prompt(session: AgentSession) -> None: + session.cancel_event.set() + active_task = session.active_prompt_task + if active_task is None or active_task is asyncio.current_task(): + return + try: + await asyncio.shield(active_task) + except Exception: # noqa: BLE001 + logger.exception( + "Active ACP prompt failed while the session was being replaced or closed." + ) + + @classmethod + async def _stop_session(cls, session: AgentSession) -> None: + await cls._stop_active_prompt(session) + if session.mcp_tools is not None: + await asyncio.to_thread(session.mcp_tools.close) + + async def initialize( + self, + protocol_version: int, + client_capabilities: object | None = None, + **kwargs: Any, + ) -> InitializeResponse: + negotiated_protocol_version = ( + protocol_version + if protocol_version in SUPPORTED_PROTOCOL_VERSIONS + else max(SUPPORTED_PROTOCOL_VERSIONS) + ) + self._client_capabilities = client_capabilities or kwargs.get( + "clientCapabilities" + ) + async_runner = EventLoopAsyncRunner(asyncio.get_running_loop()) + acp_file_backend = build_acp_file_system_backend( + client=self._conn, + session_id=config.session_id, + client_capabilities=self._client_capabilities, + async_runner=async_runner, + ) + acp_command_backend = build_acp_command_backend( + client=self._conn, + session_id=config.session_id, + client_capabilities=self._client_capabilities, + async_runner=async_runner, + ) + self._bootstrap_core.rebuild_tools( + acp_file_backend=acp_file_backend, + acp_command_backend=acp_command_backend, + ) + self.file_tools = self._bootstrap_core.file_tools + self.command_tools = self._bootstrap_core.command_tools + initialize_auth_succeeded = False + try: + if not getattr(self._authorization, "needs_workspace", False): + await self._authenticate_if_needed() + initialize_auth_succeeded = True + except RequestError: + # Some ACP clients never invoke the custom auth handshake for local agents. + # Try once during initialize, but keep later session/prompt calls fail-closed. + pass + _record_acp_runtime_event( + self._bootstrap_core._telemetry, + event_type="agent.acp.initialize", + session_id=config.session_id, + request_id=uuid4().hex, + payload={ + "client_capabilities": _normalize_capabilities( + self._client_capabilities + ), + "authorization_bootstrap_succeeded": initialize_auth_succeeded, + "file_backend": "acp" if acp_file_backend is not None else "local", + "command_backend": ( + "acp" if acp_command_backend is not None else "local" + ), + "read_text_file_enabled": bool( + getattr(acp_file_backend, "read_text_file_enabled", False) + ), + "write_text_file_enabled": bool( + getattr(acp_file_backend, "write_text_file_enabled", False) + ), + "terminal_enabled": bool( + getattr(acp_command_backend, "terminal_enabled", False) + ), + }, + ) + return InitializeResponse( + protocol_version=negotiated_protocol_version, + agent_capabilities=AgentCapabilities( + # session/load replays the conversation from the persisted + # model memory (user and agent messages, completed tool cards). + loadSession=True, + promptCapabilities=PromptCapabilities( + image=False, + audio=False, + embeddedContext=True, + ), + sessionCapabilities=SessionCapabilities( + resume=SessionResumeCapabilities(), + close=SessionCloseCapabilities(), + ), + mcpCapabilities=McpCapabilities( + http=True, + sse=True, + ), + ), + agent_info=Implementation( + name="code4me2-agent", + title="Code4Me Agent", + version="0.1.0", + ), + auth_methods=[ + AuthMethodAgent( + id="code4me-plugin-session", + name="Code4Me Plugin Session", + description="Authenticate using a prepared Code4Me project session.", + ) + ], + ) + + async def authenticate( + self, method_id: str, **kwargs: Any + ) -> AuthenticateResponse: + if method_id != "code4me-plugin-session": + raise RequestError.auth_required({"reason": "unsupported_auth_method"}) + if config.managed_mode: + # Workspace-scoped authentication completes at session/new. + return AuthenticateResponse() + await self._authenticate_if_needed() + return AuthenticateResponse() + + async def _authenticate_if_needed(self, *, force: bool = False) -> None: + if ( + not force + and self._authorization.is_authenticated + and self._authorized_workspace is not None + ): + logger.info( + "ACP runtime already has authenticated workspace=%s.", + self._authorized_workspace, + ) + return + # Retry with delay: the plugin may need time to write a fresh ACP handoff + # after a full restart (Docker + IDE) before the agent can use it. + max_attempts = 5 + for attempt in range(1, max_attempts + 1): + try: + logger.info( + "ACP runtime starting backend authentication (attempt %d/%d).", + attempt, + max_attempts, + ) + scope = await asyncio.to_thread(self._authorization.authenticate) + except AcpAuthorizationFailure as exc: + if attempt < max_attempts: + logger.warning( + "ACP runtime backend authentication rejected (attempt %d/%d): %s.", + attempt, + max_attempts, + exc, + ) + await asyncio.sleep(2.0) + continue + logger.warning( + "ACP runtime backend authentication rejected after %d attempts: %s.", + max_attempts, + exc, + ) + raise RequestError.auth_required( + {"reason": "authorization_rejected"} + ) from None + break + self._authorized_workspace = Path(scope.workspace).resolve() + logger.info( + "ACP runtime authorized workspace=%s project_id=%s.", + self._authorized_workspace, + scope.project_id, + ) + self._apply_server_config() + + def _apply_server_config(self) -> None: + """Adopt the assigned agent profile handed down by the backend. + + The whole merge of provider settings lives in + ``AgentConfig.with_server_overrides`` so there's one place that + decides precedence (server wins, local file is fallback) and one + place that decides routing (direct provider vs backend relay). + """ + server_config = self._authorization.server_agent_config + if server_config is None or not server_config.has_overrides: + logger.info("No server agent config overrides to apply.") + return + + base = self._current_config + backend_url = getattr(self._authorization, "backend_url", None) + new_config = base.with_server_overrides( + server_config, backend_url=backend_url + ) + self._current_config = new_config + self._apply_config_to_all_cores(new_config) + + provider = new_config.adapter.provider + logger.info( + "Applied server agent config: profile=%s model=%s provider_kind=%s " + "base_url=%s max_iterations=%s commands_denylist=%s.", + server_config.agent_profile, + provider.model, + provider.kind, + provider.base_url, + new_config.adapter.max_iterations, + new_config.commands.blocked_commands, + ) + if not provider.is_configured: + # Loud, because the alternative is a confusing failure deep in + # the first model call. + logger.warning( + "Agent provider is still incompletely configured " + "(base_url=%r model=%r) — model calls will fail until the " + "assigned profile supplies both.", + provider.base_url, + provider.model, + ) + + def _apply_config_to_all_cores(self, new_config: Any) -> None: + # Rebuild every enforcement point (ToolRegistry allowed_tools / + # approval, blocked commands, adapter provider) rather than + # mutating _config in place, which would leave already-constructed + # adapters enforcing stale policy: apply_config refreshes allowed + # tools plus approval/command/provider state. + self._bootstrap_core.apply_config(new_config) + for session in self._sessions.values(): + session.core.apply_config(new_config) + logger.info( + "Updated %d active session(s) with server agent config.", + len(self._sessions), + ) + + async def _require_authenticated(self) -> None: + if ( + not self._authorization.is_authenticated + or self._authorized_workspace is None + ): + await self._authenticate_if_needed() + if ( + not self._authorization.is_authenticated + or self._authorized_workspace is None + ): + raise RequestError.auth_required() + + async def new_session( + self, + cwd: str, + mcp_servers: list[Any] | None = None, + additional_directories: list[str] | None = None, + **kwargs: Any, + ) -> NewSessionResponse: + self._reject_unsupported_session_inputs( + mcp_servers=mcp_servers, + additional_directories=additional_directories, + ) + session_id = str( + kwargs.get("session_id") or kwargs.get("sessionId") or uuid4().hex + ) + await self._create_or_resume_session( + cwd=cwd, + session_id=session_id, + event_type="agent.acp.session_created", + mcp_servers=mcp_servers, + ) + self._announce_commands_soon(session_id) + return NewSessionResponse(session_id=session_id) + + async def load_session( + self, + cwd: str, + session_id: str, + mcp_servers: list[Any] | None = None, + additional_directories: list[str] | None = None, + **kwargs: Any, + ) -> LoadSessionResponse: + """Reopen a chat: restore model memory and replay it to the client.""" + self._reject_unsupported_session_inputs( + mcp_servers=mcp_servers, + additional_directories=additional_directories, + ) + session = await self._create_or_resume_session( + cwd=cwd, + session_id=session_id, + event_type="agent.acp.session_resumed", + mcp_servers=mcp_servers, + ) + await self._replay_session(session) + self._announce_commands_soon(session_id) + return LoadSessionResponse() + + def _announce_commands_soon(self, session_id: str) -> None: + """Send ``available_commands_update`` right after the session response. + + The SDK enqueues the response before the calling task yields, so a + task created here always reaches the client after it. + """ + + async def announce() -> None: + try: + await self._conn.session_update( + session_id=session_id, + update=self._updates.available_commands( + slash_commands.available(managed=config.managed_mode) + ), + source="code4me2_agent", + ) + except Exception: # noqa: BLE001 - optional UI hint + logger.debug("Could not send available commands.", exc_info=True) + + task = asyncio.get_running_loop().create_task(announce()) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) + + async def _replay_session(self, session: AgentSession) -> None: + messages = session.core.session_memory_snapshot() or [] + updates = _replay_updates( + messages, + updates=self._updates, + workspace_root=session.core._config.workspace_root, + ) + for update in updates: + try: + await self._conn.session_update( + session_id=session.session_id, + update=update, + source="code4me2_agent", + ) + except Exception: # noqa: BLE001 - replay is best effort + logger.warning("Session replay update failed; continuing.", exc_info=True) + return + session.title = _session_title(messages) + if session.title: + try: + await self._conn.session_update( + session_id=session.session_id, + update=self._updates.session_info(title=session.title), + source="code4me2_agent", + ) + except Exception: # noqa: BLE001 + logger.debug("Could not send the session title.", exc_info=True) + + async def resume_session( + self, + cwd: str, + session_id: str, + mcp_servers: list[Any] | None = None, + additional_directories: list[str] | None = None, + **kwargs: Any, + ) -> ResumeSessionResponse: + self._reject_unsupported_session_inputs( + mcp_servers=mcp_servers, + additional_directories=additional_directories, + ) + await self._create_or_resume_session( + cwd=cwd, + session_id=session_id, + event_type="agent.acp.session_resumed", + mcp_servers=mcp_servers, + ) + self._announce_commands_soon(session_id) + return ResumeSessionResponse() + + @staticmethod + def _reject_unsupported_session_inputs( + *, + mcp_servers: list[Any] | None, + additional_directories: list[str] | None, + ) -> None: + if additional_directories: + raise RequestError.invalid_params( + {"reason": "additional_directories_not_supported"} + ) + for server in mcp_servers or []: + if not _is_supported_stdio_mcp_server(server): + raise RequestError.invalid_params( + {"reason": "mcp_servers_not_supported"} + ) + + async def close_session( + self, + session_id: str, + **kwargs: Any, + ) -> None: + session = self._sessions.get(session_id) + if session is None: + raise RequestError.invalid_params({"reason": "unknown_session"}) + try: + await self._stop_session(session) + finally: + self._sessions.pop(session_id, None) + + async def cancel( + self, + session_id: str, + **kwargs: Any, + ) -> None: + session = self._sessions.get(session_id) + if session is not None and session.active_prompt_task is not None: + session.cancel_event.set() + + async def prompt( + self, + prompt: list[Any], + session_id: str, + message_id: str | None = None, + **kwargs: Any, + ) -> PromptResponse: + request_id = message_id or uuid4().hex + session = self._sessions.get(session_id) + if session is None: + raise RequestError.auth_required({"reason": "unknown_session"}) + session_authorization = session.authorization or self._authorization + reauthenticated = False + try: + await asyncio.to_thread(session_authorization.validate) + except AcpAuthorizationFailure: + if config.managed_mode: + try: + await asyncio.to_thread( + session_authorization.prepare_workspace, + session.core._config.workspace_root, + ) + await asyncio.to_thread(session_authorization.authenticate) + reauthenticated = True + except AcpAuthorizationFailure: + raise RequestError.auth_required( + {"reason": "authorization_rejected"} + ) from None + else: + self._authorized_workspace = None + raise RequestError.auth_required( + {"reason": "authorization_rejected"} + ) from None + if reauthenticated: + refreshed_config = replace( + session.core._config, + upload=replace( + session.core._config.upload, + auth_headers=session_authorization.telemetry_headers(), + ), + ) + session.core.apply_config(refreshed_config) + async with session.prompt_lock: + session.cancel_event.clear() + session.active_prompt_task = asyncio.current_task() + try: + if _should_record_runtime_context( + client_capabilities=self._client_capabilities, + session_lookup="existing", + ): + _record_acp_runtime_event( + session.core._telemetry, + event_type="agent.acp.prompt_context", + session_id=session_id, + request_id=request_id, + payload={ + "session_lookup": "existing", + "known_session_ids": sorted(self._sessions.keys()), + "client_capabilities": _normalize_capabilities( + self._client_capabilities + ), + **_prompt_backend_state(session), + }, + ) + session_config = session.core._config + prompt_text = await _prompt_text_async( + prompt, + workspace_root=session_config.workspace_root, + attachment_budget_chars=session_config.adapter.memory_window.max_tokens, + ) + logger.info("Prompt received (%d characters).", len(prompt_text)) + run_id = uuid4().hex + if config.managed_mode: + run_payload = await asyncio.to_thread( + session_authorization.create_managed_run, + run_id=run_id, + session_id=session_id, + ) + from code4me2_agent.config import ServerAgentConfig + + try: + run_policy = ServerAgentConfig.from_managed_payload(run_payload) + except ValueError: + raise RequestError.invalid_params( + {"reason": "managed_run_policy_invalid"} + ) from None + session.core.apply_config( + session.core._config.with_server_overrides( + run_policy, + backend_url=session_authorization.backend_url, + ) + ) + result = await asyncio.to_thread( + session.core.handle_prompt, + prompt_text, + request_id, + message_id, + run_id, + session.cancel_event, + ) + await asyncio.to_thread(self._save_persisted_memory, session) + if result.final_response and not getattr(result, "response_emitted", False): + await self._conn.session_update( + session_id=session_id, + update=self._updates.agent_message( + result.final_response, + message_id=request_id, + metadata={ + "code4me2": { + "phase": "completed", + "durationMs": result.duration_ms, + } + }, + ), + source="code4me2_agent", + ) + usage = getattr(result, "usage", None) + return PromptResponse( + stop_reason=_acp_stop_reason(result.stop_reason), + usage=Usage( + total_tokens=int(usage.get("total_tokens", 0)), + input_tokens=int(usage.get("prompt_tokens", 0)), + output_tokens=int(usage.get("completion_tokens", 0)), + ) + if isinstance(usage, dict) + else None, + field_meta={ + "code4me2": { + "outcome": result.run_status, + "adapterStopReason": result.stop_reason, + "durationMs": result.duration_ms, + } + }, + ) + finally: + session.active_prompt_task = None + session.cancel_event.clear() + + return Code4MeEchoAgent() + + +def _title_from_prompt(text: str) -> str | None: + first = " ".join((text or "").split()) + if not first or first.startswith("/"): + return None + return first if len(first) <= 60 else first[:57].rstrip() + "..." + + +def _session_title(messages: list[dict[str, Any]]) -> str | None: + for message in messages: + if message.get("role") == "user" and not message.get("code4me_runtime"): + title = _title_from_prompt(str(message.get("content") or "")) + if title: + return title + return None + + +def _replay_updates( + messages: list[dict[str, Any]], + *, + updates: AcpUpdateBuilder, + workspace_root: Path, +) -> list[Any]: + """Client-visible history rebuilt from the persisted model memory. + + User prompts and agent messages replay as message chunks and every tool + call as a completed (or failed) card. Runtime-internal notes (review + findings, verification notices, checkpoints, never-run calls) are not + replayed. Results are paired with the assistant message that issued them: + providers that omit call ids repeat the fallback ids every step. + """ + from code4me2_agent.adapters import _safe_tool_event_metadata, _tool_result_summary + from code4me2_agent.tool_catalog import NO_CARD_TOOLS + + replay: list[Any] = [] + card_ids: dict[str, int] = {} + index = 0 + while index < len(messages): + message = messages[index] + role = message.get("role") + content = str(message.get("content") or "") + index += 1 + if role == "tool": + continue # consumed with the assistant message that issued it + if role == "assistant" and message.get("tool_calls"): + results: dict[str, dict[str, Any]] = {} + while index < len(messages) and messages[index].get("role") == "tool": + try: + parsed = json.loads(str(messages[index].get("content") or "{}")) + except json.JSONDecodeError: + parsed = {} + results[str(messages[index].get("tool_call_id", ""))] = ( + parsed if isinstance(parsed, dict) else {} + ) + index += 1 + if message.get("code4me_runtime") == "loop_stop": + continue + if message.get("code4me_runtime"): + continue + if role == "user" and content.strip(): + replay.append(updates.user_message(_typed_prompt_text(content))) + continue + if role != "assistant": + continue + if content.strip(): + replay.append(updates.agent_message(content)) + for call in message.get("tool_calls") or []: + if not isinstance(call, dict): + continue + name = str(call.get("name", "")) + call_id = str(call.get("id", "")) + if not name or not call_id or name in NO_CARD_TOOLS: + continue + arguments = call.get("arguments") if isinstance(call.get("arguments"), dict) else {} + metadata = _safe_tool_event_metadata(name, arguments, workspace_root=workspace_root) + if metadata is None: + continue + result = results.get(call_id, {}) + failed = result.get("status") not in (None, "ok") + summary = _tool_result_summary(name, result) if not failed else None + text = summary or ( + str(result.get("error") or result.get("reason") or result.get("status") or "") + if failed + else metadata.get("content_text") + ) + # Card ids must be unique within the replayed chat. + seen = card_ids.get(call_id, 0) + card_ids[call_id] = seen + 1 + replay.append( + updates.start_tool_call( + tool_call_id=call_id if seen == 0 else f"{call_id}~{seen + 1}", + title=metadata["title"], + kind=metadata["kind"], + status="failed" if failed else "completed", + path=metadata.get("path"), + locations=metadata.get("locations"), + content=[updates.text_tool_content(text)] if text else None, + raw_input=metadata.get("raw_input"), + ) + ) + return replay + + +def _resolve_session_cwd(cwd: str) -> Path: + workspace_root = Path(cwd).expanduser() + if not workspace_root.is_absolute(): + raise ValueError("ACP session cwd must be an absolute path.") + return workspace_root.resolve() + + +async def run_acp_stdio(config: AgentConfig) -> None: + try: + from acp import run_agent + except ImportError as exc: + raise SystemExit( + "The ACP SDK is required to run stdio mode. " + "Install this package from source with agent-client-protocol available." + ) from exc + + await run_agent( + create_acp_agent(config), + use_unstable_protocol=_ENABLE_STABLE_SESSION_ROUTE_WORKAROUND, + ) diff --git a/src/code4me2_agent/acp_updates.py b/src/code4me2_agent/acp_updates.py new file mode 100644 index 0000000..1175aba --- /dev/null +++ b/src/code4me2_agent/acp_updates.py @@ -0,0 +1,327 @@ +from __future__ import annotations + +from typing import Any, Sequence + +# The "Revise…" permission option. The research proxy and analytics read this +# literal (research.telemetry.chat_lifecycle.REVISE_OPTION_ID); keep them equal. +REVISE_OPTION_ID = "revise" +# Longest instructions the "Revise…" form accepts. +REVISE_INSTRUCTIONS_MAX_CHARS = 4000 + + +class AcpUpdateBuilder: + """Small compatibility layer over the pinned ACP Python SDK update surface.""" + + def __init__(self) -> None: + import acp + from acp import schema + + self._acp = acp + self._schema = schema + + def agent_message( + self, + text: str, + *, + message_id: str | None = None, + metadata: dict[str, Any] | None = None, + ) -> Any: + text_block = getattr(self._acp, "text_block", None) + update_agent_message = getattr(self._acp, "update_agent_message", None) + if text_block is not None and update_agent_message is not None: + update = update_agent_message(text_block(text)) + else: + update = self._schema.AgentMessageChunk( + session_update="agent_message_chunk", + content=self._schema.TextContentBlock(type="text", text=text), + ) + model_updates: dict[str, Any] = {} + if message_id is not None: + model_updates["message_id"] = message_id + if metadata is not None: + model_updates["field_meta"] = metadata + if not model_updates: + return update + return update.model_copy(update=model_updates) + + def user_message(self, text: str, *, message_id: str | None = None) -> Any: + """A replayed user message (``session/load``).""" + update = self._schema.UserMessageChunk( + session_update="user_message_chunk", + content=self._schema.TextContentBlock(type="text", text=text), + ) + if message_id is None: + return update + return update.model_copy(update={"message_id": message_id}) + + def available_commands(self, commands: Sequence[Any]) -> Any: + """``available_commands_update`` from ``slash_commands.SlashCommand`` entries.""" + available = [] + for command in commands: + hint = getattr(command, "hint", None) + available.append( + self._schema.AvailableCommand( + name=str(command.name), + description=str(command.description), + input=( + self._schema.AvailableCommandInput( + self._schema.UnstructuredCommandInput(hint=str(hint)) + ) + if hint + else None + ), + ) + ) + return self._schema.AvailableCommandsUpdate( + session_update="available_commands_update", + available_commands=available, + ) + + def session_info(self, *, title: str | None, updated_at: str | None = None) -> Any: + return self._schema.SessionInfoUpdate( + session_update="session_info_update", + title=title, + updated_at=updated_at, + ) + + def agent_thought(self, text: str, *, metadata: dict[str, Any] | None = None) -> Any: + text_block = getattr(self._acp, "text_block", None) + update_agent_thought = getattr(self._acp, "update_agent_thought", None) + if text_block is not None and update_agent_thought is not None: + update = update_agent_thought(text_block(text)) + else: + update = self._schema.AgentThoughtChunk( + session_update="agent_thought_chunk", + content=self._schema.TextContentBlock(type="text", text=text), + ) + if metadata is None: + return update + return update.model_copy(update={"field_meta": metadata}) + + def agent_plan(self, entries: Sequence[Any]) -> Any: + """Build a ``plan`` session update from PlanEntrySpec-like objects or dicts.""" + plan_entries = [] + for entry in entries: + if isinstance(entry, dict): + content = str(entry.get("content", "")) + status = str(entry.get("status", "pending")) + priority = str(entry.get("priority", "medium")) + else: + content = str(getattr(entry, "content", "")) + status = str(getattr(entry, "status", "pending")) + priority = str(getattr(entry, "priority", "medium")) + plan_entries.append( + self._schema.PlanEntry(content=content, priority=priority, status=status) + ) + return self._schema.AgentPlanUpdate(session_update="plan", entries=plan_entries) + + def usage_update( + self, + *, + used: int, + size: int, + metadata: dict[str, Any] | None = None, + ) -> Any: + update = self._schema.UsageUpdate( + session_update="usage_update", + used=max(0, int(used)), + size=max(0, int(size)), + ) + if metadata is None: + return update + return update.model_copy(update={"field_meta": metadata}) + + def text_tool_content(self, text: str) -> Any: + text_block = getattr(self._acp, "text_block", None) + tool_content = getattr(self._acp, "tool_content", None) + if text_block is not None and tool_content is not None: + return tool_content(text_block(text)) + return self._schema.ContentToolCallContent( + type="content", + content=self._schema.TextContentBlock(type="text", text=text), + ) + + def diff_tool_content(self, path: str, *, new_text: str, old_text: str | None = None) -> Any: + tool_diff_content = getattr(self._acp, "tool_diff_content", None) + if tool_diff_content is not None: + return tool_diff_content(path, new_text, old_text) + return self._schema.FileEditToolCallContent( + type="diff", + path=path, + old_text=old_text, + new_text=new_text, + ) + + def start_tool_call( + self, + *, + tool_call_id: str, + title: str, + kind: str, + status: str = "pending", + path: str | None = None, + locations: Sequence[str] | None = None, + content: Sequence[Any] | None = None, + raw_input: Any | None = None, + raw_output: Any | None = None, + ) -> Any: + location_paths = [str(item) for item in locations if item] if locations else ([path] if path else []) + location_models = ( + [self._schema.ToolCallLocation(path=item) for item in location_paths] + if location_paths + else None + ) + if raw_input is None and path is not None: + raw_input = {"path": path} + + start_tool_call = getattr(self._acp, "start_tool_call", None) + if start_tool_call is not None: + return start_tool_call( + tool_call_id, + title, + kind=kind, + status=status, + content=content, + locations=location_models, + raw_input=raw_input, + raw_output=raw_output, + ) + return self._schema.ToolCallStart( + session_update="tool_call", + tool_call_id=tool_call_id, + title=title, + kind=kind, + status=status, + content=list(content) if content is not None else None, + locations=location_models, + raw_input=raw_input, + raw_output=raw_output, + ) + + def update_tool_call( + self, + *, + tool_call_id: str, + title: str | None = None, + kind: str | None = None, + status: str | None = None, + content: Sequence[Any] | None = None, + raw_input: Any | None = None, + raw_output: Any | None = None, + ) -> Any: + update_tool_call = getattr(self._acp, "update_tool_call", None) + if update_tool_call is not None: + return update_tool_call( + tool_call_id, + title=title, + kind=kind, + status=status, + content=content, + raw_input=raw_input, + raw_output=raw_output, + ) + return self._schema.ToolCallProgress( + session_update="tool_call_update", + tool_call_id=tool_call_id, + title=title, + kind=kind, + status=status, + content=list(content) if content is not None else None, + raw_input=raw_input, + raw_output=raw_output, + ) + + def permission_request( + self, + *, + session_id: str, + tool_call_id: str, + title: str, + kind: str, + summary: str, + raw_input: Any | None = None, + session_option_name: str = "Allow for session", + content: Sequence[Any] | None = None, + revise: bool = False, + ) -> Any: + options = [ + self._schema.PermissionOption( + option_id="allow_once", + name="Allow once", + kind="allow_once", + ), + self._schema.PermissionOption( + option_id="allow_session", + name=session_option_name, + kind="allow_always", + ), + self._schema.PermissionOption( + option_id="reject_once", + name="Reject", + kind="reject_once", + ), + ] + if revise: + # A reject for this call that asks for a revised proposal (see + # revision_form); last, so the usual options keep their places. + options.append( + self._schema.PermissionOption( + option_id=REVISE_OPTION_ID, + name="Revise…", + kind="reject_once", + ) + ) + return self._schema.RequestPermissionRequest( + session_id=session_id, + tool_call=self.update_tool_call( + tool_call_id=tool_call_id, + title=title, + kind=kind, + status="pending", + content=content or [self.text_tool_content(summary)], + raw_input=raw_input, + ), + options=options, + ) + + def revision_form( + self, + *, + session_id: str, + tool_call_id: str, + hunk_labels: Sequence[str] = (), + ) -> Any: + """The ``elicitation/create`` form mode behind "Revise…" for one tool call. + + ``keep`` lists the proposed change's hunks (option ``const`` = position + in ``hunk_labels``, none ticked by default) and is left out without + labels; ``instructions`` is required. + """ + properties: dict[str, Any] = {} + if hunk_labels: + properties["keep"] = self._schema.ElicitationMultiSelectPropertySchema( + type="array", + title="Parts to keep", + description="Ticked parts are applied as proposed; the agent revises the rest.", + items=self._schema.TitledMultiSelectItems( + any_of=[ + self._schema.EnumOption(const=str(index), title=str(label)) + for index, label in enumerate(hunk_labels) + ] + ), + default=[], + ) + properties["instructions"] = self._schema.ElicitationStringPropertySchema( + type="string", + title="What should change?", + min_length=1, + max_length=REVISE_INSTRUCTIONS_MAX_CHARS, + ) + return self._schema.ElicitationFormSessionMode( + session_id=session_id, + tool_call_id=tool_call_id or None, + requested_schema=self._schema.ElicitationSchema( + properties=properties, + required=["instructions"], + ), + ) diff --git a/src/code4me2_agent/acp_utils.py b/src/code4me2_agent/acp_utils.py new file mode 100644 index 0000000..c9d6bdb --- /dev/null +++ b/src/code4me2_agent/acp_utils.py @@ -0,0 +1,9 @@ +from __future__ import annotations + + +def capability_value(capabilities: object | None, name: str) -> object | None: + if capabilities is None: + return None + if isinstance(capabilities, dict): + return capabilities.get(name) + return getattr(capabilities, name, None) diff --git a/src/code4me2_agent/adapters.py b/src/code4me2_agent/adapters.py new file mode 100644 index 0000000..c6b26b6 --- /dev/null +++ b/src/code4me2_agent/adapters.py @@ -0,0 +1,5838 @@ +from __future__ import annotations + +import hashlib +import inspect +import json +import logging +import os +import platform +import random +import re +import shlex +import uuid +from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict, dataclass, field, is_dataclass +from datetime import date +from email.utils import parsedate_to_datetime +from pathlib import Path, PurePath +from threading import Event, Thread +from time import perf_counter, sleep, time +from typing import TYPE_CHECKING, Any, Callable, Protocol, Sequence, TypeVar +from urllib.parse import urlparse + +import httpx +from openai import APIConnectionError, APIStatusError, APITimeoutError, OpenAI + +from code4me2_agent import prompting, slash_commands, tool_catalog +from code4me2_agent.config import HarnessOptions +from code4me2_agent.events import ( + AssistantTextEvent, + NoopAgentEventSink, + PlanEntrySpec, + PlanEvent, + ThoughtEvent, + ToolCallEvent, + UsageEvent, + emit_event, +) +from code4me2_agent.file_tools import TextEdit, apply_text_edits, describe_strategy +from code4me2_agent.hunks import Hunk, RevisionOffer, apply_hunks +from code4me2_agent.patching import PatchError, parse_patch +from code4me2_agent.session_state import SessionToolState, normalize_workspace_path +from code4me2_agent.tool_catalog import ( + approval_kind, + requires_manual_approval, + tool_kind, +) +from code4me2_agent.tool_errors import ( + ToolArgumentError, + ToolError, + ToolFileNotFoundError, +) +from code4me2_agent.tls import USER_AGENT + +try: + import fcntl +except ImportError: # Windows + fcntl = None # type: ignore[assignment] + +if TYPE_CHECKING: + from code4me2_agent.command_tools import WorkspaceCommandTools + from code4me2_agent.config import AgentConfig + from code4me2_agent.events import AgentEventSink + from code4me2_agent.file_tools import WorkspaceFileTools + from code4me2_agent.mcp_tools import StdioMcpToolBroker + from code4me2_agent.telemetry import AgentTelemetryRecorder + +logger = logging.getLogger(__name__) + +T = TypeVar("T") + +# Safety net for tool output entering the conversation; tools cap their own +# output well below this. +_MAX_TOOL_RESULT_CHARS = 32_000 +# Room for the model's own output when sizing the request window. +_OUTPUT_HEADROOM_TOKENS = 1024 +# Current-turn compaction elides down to this share of the budget at once. +_CURRENT_TURN_LOW_WATER = 0.6 +_MAX_TOKEN_SCALE = 3.0 +_BUDGET_NOTICE = ( + "This is the final model call for this turn: tool calls are disabled. Reply with a " + "user-facing message that summarizes what you did and verified so far, and what " + "remains to be done." +) +_EMPTY_RESPONSE_NUDGE = ( + "Your previous response was empty. Reply with user-visible text or call a tool." +) +_EMPTY_RESPONSE_TEXT = "The model returned an empty response twice; please try again." +_PARSE_ERROR_TEXT = "The model returned an unreadable response; please try again." + + +@dataclass(frozen=True) +class AdapterResult: + final_response: str + stop_reason: str + run_status: str + thoughts: tuple[str, ...] = () + # True when the adapter already streamed the final text to the client. + response_emitted: bool = False + usage: dict[str, int] | None = None + # Optional model calls (self-review, summary) that failed this turn. They + # have no closing event of their own; agent.run.completed reports them. + side_call_failures: tuple[dict[str, Any], ...] = () + + +class AgentAdapter(Protocol): + def handle_prompt( + self, + *, + prompt: str, + run_id: str, + request_id: str, + message_id: str | None, + memory: "MemoryWindow | None" = None, + cancellation_event: Event | None = None, + ) -> AdapterResult: ... + + +@dataclass(frozen=True) +class ToolCall: + tool_call_id: str + name: str + arguments: dict[str, Any] + # Set when the provider returned arguments that were not a JSON object. + argument_error: str | None = None + raw_arguments: str | None = None + + +@dataclass(frozen=True) +class ParsedProviderOutput: + text: str | None + tool_calls: list[ToolCall] + thought: str | None = None + finish_reason: str | None = None + # The provider's own reasoning fields, exactly as returned, to send back. + reasoning_fields: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class ProviderTurn: + output: dict[str, Any] + usage: dict[str, int] + finish_reason: str | None + model: str + request_payload: dict[str, Any] | None = None + raw_response: dict[str, Any] | None = None + usage_estimated: bool = False + + +class FakeProviderExhaustedError(RuntimeError): + pass + + +class ProviderRequestFailed(RuntimeError): + def __init__( + self, + message: str, + *, + status_code: int | None = None, + attempts: int = 1, + retryable: bool = False, + ) -> None: + super().__init__(message) + self.status_code = status_code + self.attempts = attempts + self.retryable = retryable + + +# Backwards-compatible alias. +BackendProviderRequestError = ProviderRequestFailed + + +class ProviderCancelled(RuntimeError): + pass + + +class ToolRegistryError(RuntimeError): + def __init__(self, message: str, *, failure_reason: str) -> None: + super().__init__(message) + self.failure_reason = failure_reason + + +class ToolRevisionRequested(ToolRegistryError): + """The user chose "Revise…": the call did not run as proposed. + + ``result`` is the tool result the model sees instead (status ``revise``). + """ + + def __init__(self, message: str, *, result: dict[str, Any]) -> None: + super().__init__(message, failure_reason="approval_revised") + self.result = result + + +def _turn_was_cancelled(cancellation_event: Event | None) -> bool: + return cancellation_event is not None and cancellation_event.is_set() + + +def _cancelled_result( + thoughts: list[str] | tuple[str, ...] = (), +) -> AdapterResult: + return AdapterResult( + final_response="", + stop_reason="cancelled", + run_status="cancelled", + thoughts=tuple(thoughts), + ) + + +# ------------------------------------------------------------ rate limiting + + +def _rate_limit_provider_request_from_env() -> None: + state_path_text = os.getenv("CODE4ME_RATE_LIMIT_STATE_PATH", "").strip() + if not state_path_text: + return + limit = _positive_int_env("CODE4ME_MODEL_REQUEST_LIMIT", 5) + window_seconds = _positive_float_env("CODE4ME_MODEL_REQUEST_WINDOW_SECONDS", 70.0) + _rate_limit_provider_request( + state_path=Path(state_path_text).expanduser(), + limit=limit, + window_seconds=window_seconds, + ) + + +def _rate_limit_provider_request( + *, + state_path: Path, + limit: int, + window_seconds: float, + now_fn=time, + sleep_fn=sleep, +) -> None: + state_path.parent.mkdir(parents=True, exist_ok=True) + lock_path = state_path.with_suffix(state_path.suffix + ".lock") + with lock_path.open("a+", encoding="utf-8") as lock_file: + if fcntl is not None: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + while True: + now = float(now_fn()) + timestamps = _read_rate_limit_timestamps(state_path) + timestamps = sorted( + timestamp + for timestamp in timestamps + if now - timestamp < window_seconds + ) + if len(timestamps) < limit: + timestamps.append(now) + _write_rate_limit_timestamps(state_path, timestamps) + return + sleep_seconds = max(0.0, timestamps[0] + window_seconds - now) + logging.info( + "Model request rate limit reached: limit=%s window_seconds=%s sleeping=%.3fs", + limit, + window_seconds, + sleep_seconds, + ) + sleep_fn(sleep_seconds) + finally: + if fcntl is not None: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +def _read_rate_limit_timestamps(state_path: Path) -> list[float]: + try: + data = json.loads(state_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return [] + if not isinstance(data, dict): + return [] + raw_timestamps = data.get("timestamps", []) + if not isinstance(raw_timestamps, list): + return [] + timestamps: list[float] = [] + for value in raw_timestamps: + try: + timestamps.append(float(value)) + except (TypeError, ValueError): + continue + return timestamps + + +def _write_rate_limit_timestamps(state_path: Path, timestamps: list[float]) -> None: + state_path.write_text( + json.dumps({"timestamps": timestamps}, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def _positive_int_env(name: str, default: int) -> int: + try: + value = int(os.getenv(name, "")) + except ValueError: + return default + return value if value > 0 else default + + +def _positive_float_env(name: str, default: float) -> float: + try: + value = float(os.getenv(name, "")) + except ValueError: + return default + return value if value > 0 else default + + +# ------------------------------------------------------------------ retries + + +@dataclass(frozen=True) +class RetryPolicy: + max_attempts: int = 4 + base_delay: float = 1.0 + max_delay: float = 20.0 + max_total_wait: float = 60.0 + retry_after_cap: float = 30.0 + retry_statuses: frozenset[int] = frozenset({408, 409, 429, 500, 502, 503, 504}) + + @classmethod + def from_env(cls) -> "RetryPolicy": + return cls( + max_attempts=_positive_int_env("CODE4ME_PROVIDER_MAX_ATTEMPTS", 4), + max_total_wait=_positive_float_env("CODE4ME_PROVIDER_MAX_RETRY_SECONDS", 60.0), + ) + + +@dataclass(frozen=True) +class _ErrorClass: + retryable: bool + status_code: int | None = None + retry_after: float | None = None + message: str = "" + + +def _sleep_interruptible( + seconds: float, + cancellation_event: Event | None, + sleep_fn: Callable[[float], None], +) -> None: + remaining = max(0.0, seconds) + while remaining > 0: + if _turn_was_cancelled(cancellation_event): + raise ProviderCancelled("Cancelled while waiting to retry the model request.") + step = min(0.25, remaining) + sleep_fn(step) + remaining -= step + if _turn_was_cancelled(cancellation_event): + raise ProviderCancelled("Cancelled while waiting to retry the model request.") + + +def _call_with_retries( + call: Callable[[], T], + *, + policy: RetryPolicy, + classify: Callable[[BaseException], _ErrorClass], + cancellation_event: Event | None = None, + sleep_fn: Callable[[float], None] = sleep, + rand: Callable[[], float] = random.random, + before_attempt: Callable[[], None] | None = None, +) -> T: + total_wait = 0.0 + attempts = max(1, policy.max_attempts) + for attempt in range(1, attempts + 1): + if _turn_was_cancelled(cancellation_event): + raise ProviderCancelled("Cancelled before the model request was sent.") + if before_attempt is not None: + before_attempt() + try: + return call() + except ProviderCancelled: + raise + except Exception as exc: # noqa: BLE001 + classified = classify(exc) + description = classified.message or str(exc) + if not classified.retryable or attempt >= attempts: + raise ProviderRequestFailed( + description, + status_code=classified.status_code, + attempts=attempt, + retryable=classified.retryable, + ) from exc + if classified.retry_after is not None: + delay = min(max(0.0, classified.retry_after), policy.retry_after_cap) + else: + cap = min(policy.max_delay, policy.base_delay * (2 ** (attempt - 1))) + delay = cap * (0.5 + 0.5 * rand()) + if total_wait + delay > policy.max_total_wait: + raise ProviderRequestFailed( + f"{description} (gave up after {attempt} attempts; retry budget exhausted)", + status_code=classified.status_code, + attempts=attempt, + retryable=True, + ) from exc + logging.info( + "Model request failed (attempt %s/%s, status=%s); retrying in %.2fs", + attempt, + attempts, + classified.status_code, + delay, + ) + _sleep_interruptible(delay, cancellation_event, sleep_fn) + total_wait += delay + raise ProviderRequestFailed("Model request failed.", attempts=attempts) + + +def _parse_retry_after(value: object) -> float | None: + if value is None: + return None + text = str(value).strip() + if not text: + return None + try: + return max(0.0, float(text)) + except ValueError: + pass + try: + retry_at = parsedate_to_datetime(text) + except (TypeError, ValueError): + return None + if retry_at.tzinfo is None: + return None + from datetime import datetime, timezone + + return max(0.0, (retry_at - datetime.now(timezone.utc)).total_seconds()) + + +def _classify_sdk_error(policy: RetryPolicy) -> Callable[[BaseException], _ErrorClass]: + def classify(exc: BaseException) -> _ErrorClass: + if isinstance(exc, APIStatusError): + status = int(getattr(exc, "status_code", 0) or 0) + headers = getattr(getattr(exc, "response", None), "headers", None) + retry_after = None + if headers is not None: + try: + retry_after = _parse_retry_after(headers.get("retry-after")) + except Exception: # noqa: BLE001 + retry_after = None + body = "" + response = getattr(exc, "response", None) + if response is not None: + try: + body = str(response.text)[:500] + except Exception: # noqa: BLE001 + body = "" + return _ErrorClass( + retryable=status in policy.retry_statuses, + status_code=status, + retry_after=retry_after, + message=f"The model request failed with HTTP {status}: {body}".rstrip(": "), + ) + if isinstance(exc, APIConnectionError): + return _ErrorClass( + retryable=True, + message=f"The model request could not reach the model service: {exc}", + ) + return _ErrorClass(retryable=False, message=f"The model request failed: {exc}") + + return classify + + +def _classify_managed_error(policy: RetryPolicy) -> Callable[[BaseException], _ErrorClass]: + def classify(exc: BaseException) -> _ErrorClass: + status = getattr(exc, "status_code", None) + transient = bool(getattr(exc, "transient", False)) + retryable = transient or (isinstance(status, int) and status in policy.retry_statuses) + if status is not None: + message = f"The managed model request failed with HTTP {status}." + else: + message = f"The managed model request failed: {exc}" + return _ErrorClass( + retryable=retryable, + status_code=status if isinstance(status, int) else None, + message=message, + ) + + return classify + + +# ----------------------------------------------------------------- adapters + + +class DeterministicEchoAdapter: + def handle_prompt( + self, + *, + prompt: str, + run_id: str, + request_id: str, + message_id: str | None, + memory: "MemoryWindow | None" = None, + cancellation_event: Event | None = None, + ) -> AdapterResult: + if _turn_was_cancelled(cancellation_event): + return _cancelled_result() + return AdapterResult( + final_response=f"Code4Me ACP echo: {prompt}", + stop_reason="end_turn", + run_status="completed", + ) + + +@dataclass(frozen=True) +class EditPreview: + old_text: str | None + new_text: str | None + error: BaseException | None = None + # Additional files of a multi-file change (apply_patch): (absolute path, old, new). + extra_diffs: tuple[tuple[str, str | None, str], ...] = () + + @property + def has_diff(self) -> bool: + return self.new_text is not None + + +# Tools whose "Revise…" form can keep some hunks of the change (apply_patch +# only for a single update without a move); others get instructions only. +_REVISE_HUNK_TOOLS = frozenset({"create_file", "write_file", "replace_text", "edit_file"}) + + +@dataclass(frozen=True) +class _Revision: + """What became of an accepted "Revise…" form (``revise_status``). + + ``applied``: the kept hunks were written; ``file_changed``: the file no + longer held the previewed text, so nothing was written; ``write_failed``: + writing them failed; ``instructions_only``: no hunk was kept. + """ + + status: str + kept: tuple[Hunk, ...] + total: int + instructions: str + path: str | None = None + merged_text: str | None = None + error: str | None = None + syntax_error: dict[str, Any] | None = None + + def result(self) -> dict[str, Any]: + """The tool result the model sees instead of the tool output.""" + result: dict[str, Any] = { + "status": "revise", + "kept_hunks": [hunk.label for hunk in self.kept], + "total_hunks": self.total, + "applied_path": self.path if self.status == "applied" else None, + "user_instructions": self.instructions, + "message": self._message(), + } + if self.syntax_error: + result["syntax_error"] = self.syntax_error + return result + + def _message(self) -> str: + parts = f"{len(self.kept)} of the {self.total} parts of the proposed change" + if self.status == "applied": + done = ( + f"The user kept {parts}; they are already written to {self.path} (re-read the " + "file before changing it again). Nothing else was applied." + ) + elif self.status == "file_changed": + done = ( + f"The user kept {parts}, but {self.path} changed after the change was proposed, " + "so nothing was written; re-read the file." + ) + elif self.status == "write_failed": + done = ( + f"The user kept {parts}, but writing them to {self.path} failed ({self.error}); " + "re-read the file." + ) + else: + done = "The user did not approve this call as proposed; nothing was applied." + if self.instructions: + return ( + f"{done} The user asked for a revision: follow user_instructions and propose the " + "revised version in this turn; it needs the user's approval again." + ) + return ( + f"{done} The user gave no further instructions: do not propose the dropped parts " + "again; continue the task or ask the user what should change." + ) + + +# Tools whose success means the workspace changed; a loop-guard count resets +# after one, and verify-on-stop / self-review look at the turn's changes. +_WORKSPACE_MUTATIONS = frozenset( + {"create_file", "write_file", "replace_text", "edit_file", "apply_patch", "delete_file", "move_file"} +) +# Guarded by read-before-edit: changing an existing file needs a prior look. +_READ_BEFORE_EDIT_TOOLS = frozenset({"write_file", "replace_text", "edit_file"}) +# Built-in tools that never change anything; they may run in parallel. +_PARALLEL_SAFE_TOOLS = frozenset({"read_file", "list_files", "glob_files", "grep_files", "search_files"}) + + +class ToolRegistry: + def __init__( + self, + file_tools: WorkspaceFileTools, + command_tools: WorkspaceCommandTools, + *, + allowed_tools: frozenset[str] | list[str] | None = None, + event_sink: AgentEventSink | None = None, + mcp_tools: StdioMcpToolBroker | None = None, + approval_policy: str = "auto", + telemetry: AgentTelemetryRecorder | None = None, + workspace_root: Path | None = None, + session_state: SessionToolState | None = None, + harness: HarnessOptions | None = None, + ) -> None: + self._file_tools = file_tools + self._command_tools = command_tools + self._mcp_tools = mcp_tools + self._allowed_tools = frozenset(allowed_tools) if allowed_tools is not None else None + self._approval_policy = approval_policy + self._event_sink = event_sink or NoopAgentEventSink() + self._telemetry = telemetry + if workspace_root is None: + candidate = getattr(file_tools, "workspace_root", None) + workspace_root = candidate if isinstance(candidate, Path) else None + self._workspace_root = workspace_root + # Read-before-edit needs the session's memory of what was read; a + # registry built without one (tests, tools used directly) skips it. + self._session_state = session_state + self._harness = harness or HarnessOptions() + self._handlers: dict[str, Callable[..., Any]] = { + "read_file": self._run_read_file, + "create_file": self._run_create_file, + "write_file": self._run_write_file, + "replace_text": self._run_replace_text, + "edit_file": self._run_edit_file, + "apply_patch": self._run_apply_patch, + "delete_file": self._run_delete_file, + "move_file": self._run_move_file, + "list_files": self._run_list_files, + "glob_files": self._run_glob_files, + "grep_files": self._run_grep_files, + "search_files": self._run_search_files, + "run_command": self._run_run_command, + "update_plan": self._run_update_plan, + "ask_user": self._run_ask_user, + } + + # ------------------------------------------------------------ policy + + @property + def approval_policy(self) -> str: + return self._approval_policy + + @property + def session_state(self) -> SessionToolState | None: + return self._session_state + + @property + def command_tools(self) -> WorkspaceCommandTools: + return self._command_tools + + @property + def file_tools(self) -> WorkspaceFileTools: + return self._file_tools + + def _is_allowed(self, name: str) -> bool: + if self._allowed_tools is None: + return True + if name in self._allowed_tools: + return True + return name.startswith("mcp__") and "mcp__*" in self._allowed_tools + + def mcp_access(self, name: str) -> str | None: + """The broker's class for an MCP tool ("read", "execute", "edit", "other").""" + return self._mcp_access(name) + + def _mcp_access(self, name: str) -> str | None: + if not name.startswith("mcp__") or self._mcp_tools is None: + return None + access = getattr(self._mcp_tools, "tool_access", None) + if not callable(access): + return None + try: + return str(access(name)) + except Exception: # noqa: BLE001 + return None + + def requires_approval(self, name: str) -> bool: + """Whether ``name`` changes anything (approval under per_step, hidden under suggestion_only). + + Client-supplied MCP tools are treated as mutating unless the broker + classified them read-only (the curated IDE diagnostics and search tools). + """ + if name.startswith("mcp__") and self._mcp_access(name) == "read": + return False + return requires_manual_approval(name) + + def is_parallel_safe(self, name: str) -> bool: + if name in _PARALLEL_SAFE_TOOLS: + return True + return name.startswith("mcp__") and self._mcp_access(name) == "read" + + def display_kind(self, name: str) -> str: + access = self._mcp_access(name) + if access == "read": + return "search" + if access == "execute": + return "execute" + if access == "edit": + return "edit" + return tool_kind(name) + + def definitions(self) -> list[dict[str, Any]]: + definitions = tool_catalog.tool_definitions() + if self._mcp_tools is not None: + definitions.extend(self._mcp_tools.definitions()) + selected = [ + definition + for definition in definitions + if self._is_allowed(str(definition.get("function", {}).get("name", ""))) + ] + if self._approval_policy == "suggestion_only": + # The model must never see tools it cannot run; it proposes diffs instead. + selected = [ + definition + for definition in selected + if not self.requires_approval( + str(definition.get("function", {}).get("name", "")) + ) + ] + blocked = getattr(self._command_tools, "blocked_commands", None) + shell = _usable_shell(blocked) if blocked is not None else None + if shell is not None: + # "There is no shell" would contradict an installed, unblocked bash/sh. + selected = [_with_shell_description(definition, shell) for definition in selected] + return selected + + def _is_known(self, name: str) -> bool: + """A catalogue tool or a tool of a connected MCP server (allowed or not).""" + if name in tool_catalog.tool_names(): + return True + return self._mcp_tools is not None and self._mcp_tools.has_tool(name) + + def known_tool_names(self) -> set[str]: + names: set[str] = set() + for tool in self.definitions(): + function = tool.get("function") + if isinstance(function, dict): + name = str(function.get("name", "")).strip() + if name: + names.add(name) + return names + + def needs_approval(self, name: str) -> bool: + return self._approval_policy == "per_step" and self.requires_approval(name) + + # ------------------------------------------------------ read tracking + + def _relative(self, path: object) -> str | None: + if self._workspace_root is None: + return None + return normalize_workspace_path(self._workspace_root, path) + + def _check_read_before_edit(self, name: str, arguments: dict[str, Any]) -> None: + state = self._session_state + if state is None or not self._harness.read_before_edit or arguments.get("force"): + return + if name in _READ_BEFORE_EDIT_TOOLS: + paths = [arguments.get("path")] + elif name == "apply_patch": + paths = list(arguments.get("_guarded_paths") or []) + else: + return + for path in paths: + relative = self._relative(path) + if relative is None or state.has_seen(relative): + continue + if self._workspace_root is not None and not (self._workspace_root / relative).exists(): + continue # a new file: nothing to have read + raise ToolError( + f"{relative} has not been read in this session. Call read_file on it first so the " + "edit is based on its current content (or pass force=true if you are certain of it).", + code="read_before_edit", + ) + + def _note_success(self, name: str, arguments: dict[str, Any], output: dict[str, Any]) -> None: + state = self._session_state + if state is None: + return + if name in {"read_file", "create_file", "write_file", "replace_text", "edit_file"}: + state.mark_seen(self._relative(output.get("path") or arguments.get("path"))) + elif name == "apply_patch": + for item in output.get("files") or []: + if isinstance(item, dict) and item.get("action") != "delete": + state.mark_seen(self._relative(item.get("path"))) + elif name == "move_file": + source = self._relative(arguments.get("source_path")) + if state.has_seen(source): + state.mark_seen(self._relative(arguments.get("destination_path"))) + elif name in {"grep_files", "search_files"}: + for match in output.get("matches") or []: + if isinstance(match, dict): + state.mark_seen(self._relative(match.get("path"))) + + # ----------------------------------------------------------- execute + + def _record_permission( + self, + event_type: str, + *, + run_id: str, + request_id: str, + parent_event_id: str | None, + payload: dict[str, Any], + ) -> dict[str, Any] | None: + """Emit a permission.requested/decided telemetry event, if wired. + + Payloads carry only structural metadata (tool name/kind/ids, decision + outcome and scope) — never arguments or content — so they survive both + local redaction and the server privacy gate under metadata policies. + Returns the recorded event (for parent linkage) or ``None``. + """ + record = getattr(self._telemetry, "record", None) + if not callable(record): + return None + try: + return record( + event_type=event_type, + run_id=run_id, + request_id=request_id, + parent_event_id=parent_event_id, + payload=payload, + ) + except Exception: # noqa: BLE001 - telemetry must never break tools + return None + + def execute( + self, + tool_call: ToolCall, + *, + run_id: str, + request_id: str, + cancellation_event: Event | None = None, + ) -> dict[str, Any]: + name = tool_call.name + arguments = dict(tool_call.arguments) + metadata = _safe_tool_event_metadata(name, arguments, workspace_root=self._workspace_root) + + if not self._is_known(name): + available = sorted(self.known_tool_names()) + hint = ( + " Use run_command to run programs." + if name.lower() in _SHELL_LIKE_TOOL_NAMES and "run_command" in available + else "" + ) + message = f"Unknown tool: {name}. Available tools: {', '.join(available) or 'none'}.{hint}" + self._emit_denied(tool_call, arguments, metadata, message=message) + raise ToolRegistryError(message, failure_reason="unsupported_tool") + if not self._is_allowed(name): + message = f"Tool is disabled by the assigned study policy: {name}" + self._emit_denied(tool_call, arguments, metadata, message=message) + raise ToolRegistryError(message, failure_reason="tool_not_allowed") + if self._approval_policy not in {"auto", "per_step", "suggestion_only"}: + message = f"Unknown assigned approval policy: {self._approval_policy}" + self._emit_denied(tool_call, arguments, metadata, message=message) + raise ToolRegistryError(message, failure_reason="invalid_approval_policy") + if self._approval_policy == "suggestion_only" and self.requires_approval(name): + message = f"Tool execution is disabled by suggestion-only policy: {name}" + self._record_permission( + "agent.permission.decided", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "tool_name": name, + "tool_call_id": tool_call.tool_call_id, + "decision": "rejected", + "decision_scope": "policy", + }, + ) + self._emit_denied(tool_call, arguments, metadata, message=message) + raise ToolRegistryError(message, failure_reason="approval_policy_denied") + + try: + validated = _validate_arguments(name, arguments) + if name == "apply_patch": + validated["_patch"] = parse_patch(validated["patch"]) + validated["_guarded_paths"] = [ + change.path for change in validated["_patch"] if change.action != "add" + ] + except PatchError as exc: + self._emit_denied(tool_call, arguments, metadata, message=f"Invalid patch: {exc}") + raise ToolArgumentError(str(exc), field="patch") from None + except ToolArgumentError as exc: + self._emit_denied(tool_call, arguments, metadata, message=f"Invalid arguments: {exc}") + raise + metadata = _safe_tool_event_metadata(name, validated, workspace_root=self._workspace_root) + if metadata is not None and name.startswith("mcp__"): + metadata["kind"] = self.display_kind(name) + try: + self._check_read_before_edit(name, validated) + except ToolError as exc: + self._emit_denied(tool_call, arguments, metadata, message=str(exc)) + raise + preview = self._edit_preview(name, validated, tool_call, run_id=run_id, request_id=request_id) + pending = self.needs_approval(name) + self._emit_tool_start(tool_call, validated, metadata, preview, pending=pending) + if preview is not None and preview.error is not None: + self._emit_tool_failed(tool_call, validated, metadata, preview, error=preview.error) + raise preview.error + + if pending: + requested = self._record_permission( + "agent.permission.requested", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "tool_name": name, + "tool_call_id": tool_call.tool_call_id, + "kind": metadata["kind"] if metadata else None, + }, + ) + request_approval = getattr(self._event_sink, "request_approval", None) + offer = self._revision_offer(name, validated, preview) + if not callable(request_approval): + decision = None + elif offer is not None and _accepts_keyword(request_approval, "revise"): + decision = request_approval(tool_call, validated, revise=offer) + else: + decision = request_approval(tool_call, validated) + outcome = getattr(decision, "decision", "unavailable") + scope = getattr(decision, "scope", None) + revision = ( + self._apply_revision( + tool_call, + decision, + offer or RevisionOffer(), + run_id=run_id, + request_id=request_id, + ) + if outcome == "revised" + else None + ) + self._record_permission( + "agent.permission.decided", + run_id=run_id, + request_id=request_id, + parent_event_id=(requested or {}).get("event_id"), + payload={ + "tool_name": name, + "tool_call_id": tool_call.tool_call_id, + "kind": metadata["kind"] if metadata else None, + "decision": ( + outcome + if outcome in ("accepted", "rejected", "cancelled", "unavailable", "revised") + else "unavailable" + ), + "decision_scope": scope, + **_revision_telemetry(decision, offer, revision), + }, + ) + if revision is not None: + self._emit_revision_card(tool_call, validated, metadata, preview, offer, revision) + raise ToolRevisionRequested( + f"Tool approval revised for: {name}", result=revision.result() + ) + if not getattr(decision, "accepted", bool(decision)): + self._emit_tool_failed( + tool_call, + validated, + metadata, + preview, + message=f"Not run: approval {outcome}.", + ) + raise ToolRegistryError( + f"Tool approval {outcome} for: {name}", + failure_reason=f"approval_{outcome}", + ) + else: + # No user round-trip (auto policy or tool needs none): the policy + # itself accepted, recorded so every execution has a decision. + self._record_permission( + "agent.permission.decided", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "tool_name": name, + "tool_call_id": tool_call.tool_call_id, + "decision": "accepted", + "decision_scope": "policy", + }, + ) + + try: + handler = self._handlers.get(name) + if handler is not None: + result = handler( + tool_call, + validated, + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + ) + elif self._mcp_tools is not None and self._mcp_tools.has_tool(name): + result = self._mcp_tools.execute(name, validated) + else: + raise ToolRegistryError( + f"Unsupported tool name: {name}", + failure_reason="unsupported_tool", + ) + except Exception as exc: + self._emit_tool_failed(tool_call, validated, metadata, preview, error=exc) + raise + tool_output = asdict(result) if is_dataclass(result) else dict(result) + if validated.get("_argument_notes"): + tool_output["argument_notes"] = list(validated["_argument_notes"]) + self._note_success(name, validated, tool_output) + self._emit_tool_completed(tool_call, validated, tool_output, metadata, preview) + return tool_output + + def report_invalid_arguments( + self, + tool_call: ToolCall, + *, + run_id: str, + request_id: str, + ) -> None: + """Draw started+failed cards for a call whose arguments never parsed.""" + metadata = _safe_tool_event_metadata( + tool_call.name, dict(tool_call.arguments), workspace_root=self._workspace_root + ) + self._emit_denied( + tool_call, + dict(tool_call.arguments), + metadata, + message=f"Invalid arguments: {tool_call.argument_error}", + ) + + # ------------------------------------------------------------ revise + + def _revision_offer( + self, name: str, arguments: dict[str, Any], preview: EditPreview | None + ) -> RevisionOffer | None: + """What "Revise…" offers on this approval; None when the profile switched it off. + + A single-file text change offers its hunks; any other call offers + instructions only. + """ + if not self._harness.approval_revise: + return None + path: object = None + if preview is not None and preview.has_diff: + if name in _REVISE_HUNK_TOOLS: + path = arguments.get("path") + elif name == "apply_patch": + actions = list(arguments.get("_patch") or []) + if ( + len(actions) == 1 + and getattr(actions[0], "action", None) == "update" + and getattr(actions[0], "move_to", None) is None + ): + path = getattr(actions[0], "path", None) + if not isinstance(path, str) or not path or preview is None: + return RevisionOffer() + return RevisionOffer(path=path, old_text=preview.old_text, new_text=preview.new_text) + + def _apply_revision( + self, + tool_call: ToolCall, + decision: object, + offer: RevisionOffer, + *, + run_id: str, + request_id: str, + ) -> _Revision: + """Write the hunks the user kept, once, if the file still holds the previewed text. + + The proposed call itself never runs. Nothing is written when no hunk + was kept or the file changed since the preview. + """ + wanted = set(getattr(decision, "kept_hunks", ()) or ()) + kept = tuple(hunk for hunk in offer.selectable_hunks if hunk.index in wanted) + instructions = str(getattr(decision, "instructions", "") or "") + total = len(offer.hunks) + if not kept or offer.path is None or offer.old_text is None or offer.new_text is None: + return _Revision("instructions_only", (), total, instructions) + try: + current: str | None = self._read_current_text( + offer.path, tool_call, run_id=run_id, request_id=request_id + ) + except Exception: # noqa: BLE001 - gone or unreadable: not the previewed text + current = None + if current != offer.old_text: + return _Revision("file_changed", kept, total, instructions, path=offer.path) + merged = apply_hunks(offer.old_text, offer.new_text, [hunk.index for hunk in kept]) + write_file = self._file_tools.write_file + kwargs: dict[str, Any] = { + "path": offer.path, + "content": merged, + "tool_call_id": tool_call.tool_call_id, + "run_id": run_id, + "request_id": request_id, + } + if _accepts_keyword(write_file, "tool_name"): + kwargs["tool_name"] = tool_call.name + try: + written = write_file(**kwargs) + except Exception as exc: # noqa: BLE001 + return _Revision( + "write_failed", kept, total, instructions, path=offer.path, error=str(exc) + ) + syntax_error = getattr(written, "syntax_error", None) + return _Revision( + "applied", + kept, + total, + instructions, + path=offer.path, + merged_text=merged, + syntax_error=syntax_error if isinstance(syntax_error, dict) else None, + ) + + # ---------------------------------------------------------- handlers + + def _run_read_file(self, tool_call, args, *, run_id, request_id, cancellation_event): + kwargs: dict[str, Any] = { + "path": args["path"], + "tool_call_id": tool_call.tool_call_id, + "run_id": run_id, + "request_id": request_id, + } + if args.get("offset") is not None: + kwargs["offset"] = args["offset"] + if args.get("limit") is not None: + kwargs["limit"] = args["limit"] + return self._file_tools.read_file(**kwargs) + + def _run_create_file(self, tool_call, args, *, run_id, request_id, cancellation_event): + return self._file_tools.create_file( + path=args["path"], + content=args["content"], + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + + def _run_write_file(self, tool_call, args, *, run_id, request_id, cancellation_event): + return self._file_tools.write_file( + path=args["path"], + content=args["content"], + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + + def _run_replace_text(self, tool_call, args, *, run_id, request_id, cancellation_event): + kwargs: dict[str, Any] = { + "path": args["path"], + "old_text": args["old_text"], + "new_text": args["new_text"], + "tool_call_id": tool_call.tool_call_id, + "run_id": run_id, + "request_id": request_id, + } + if args.get("replace_all") and _accepts_keyword(self._file_tools.replace_text, "replace_all"): + kwargs["replace_all"] = True + return self._file_tools.replace_text(**kwargs) + + def _run_edit_file(self, tool_call, args, *, run_id, request_id, cancellation_event): + return self._file_tools.edit_file( + path=args["path"], + edits=args["edits"], + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + + def _run_delete_file(self, tool_call, args, *, run_id, request_id, cancellation_event): + return self._file_tools.delete_file( + path=args["path"], + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + + def _run_move_file(self, tool_call, args, *, run_id, request_id, cancellation_event): + return self._file_tools.move_file( + source_path=args["source_path"], + destination_path=args["destination_path"], + overwrite=bool(args.get("overwrite", False)), + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + + def _run_list_files(self, tool_call, args, *, run_id, request_id, cancellation_event): + kwargs: dict[str, Any] = { + "path": args.get("path", "."), + "tool_call_id": tool_call.tool_call_id, + "run_id": run_id, + "request_id": request_id, + } + if _accepts_keyword(self._file_tools.list_files, "recursive"): + kwargs.update( + recursive=bool(args.get("recursive", False)), + max_depth=args.get("max_depth"), + max_results=args.get("max_results", 500), + include_ignored=bool(args.get("include_ignored", False)), + ) + return self._file_tools.list_files(**kwargs) + + def _run_glob_files(self, tool_call, args, *, run_id, request_id, cancellation_event): + return self._file_tools.glob_files( + args["pattern"], + path=args.get("path", "."), + max_results=args.get("max_results", 500), + include_ignored=bool(args.get("include_ignored", False)), + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + + def _run_grep_files(self, tool_call, args, *, run_id, request_id, cancellation_event): + kwargs: dict[str, Any] = { + "path": args.get("path", "."), + "glob": args.get("glob"), + "case_insensitive": bool(args.get("case_insensitive", False)), + "context_lines": args.get("context_lines", 0), + "max_results": args.get("max_results", 200), + "output_mode": args.get("output_mode", "content"), + "include_ignored": bool(args.get("include_ignored", False)), + "tool_call_id": tool_call.tool_call_id, + "run_id": run_id, + "request_id": request_id, + } + if cancellation_event is not None and _accepts_keyword(self._file_tools.grep_files, "cancellation_event"): + kwargs["cancellation_event"] = cancellation_event + return self._file_tools.grep_files(args["pattern"], **kwargs) + + def _run_search_files(self, tool_call, args, *, run_id, request_id, cancellation_event): + kwargs: dict[str, Any] = { + "query": args["query"], + "path": args.get("path", "."), + "tool_call_id": tool_call.tool_call_id, + "run_id": run_id, + "request_id": request_id, + } + if cancellation_event is not None and _accepts_keyword(self._file_tools.search_files, "cancellation_event"): + kwargs["cancellation_event"] = cancellation_event + return self._file_tools.search_files(**kwargs) + + def _run_apply_patch(self, tool_call, args, *, run_id, request_id, cancellation_event): + return self._file_tools.apply_patch( + args["_patch"], + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + + def _run_run_command(self, tool_call, args, *, run_id, request_id, cancellation_event): + kwargs: dict[str, Any] = { + "argv": args["argv"], + "cwd": args.get("cwd", "."), + "tool_call_id": tool_call.tool_call_id, + "run_id": run_id, + "request_id": request_id, + } + run_command = self._command_tools.run_command + if args.get("timeout_seconds") is not None and _accepts_keyword(run_command, "timeout_seconds"): + kwargs["timeout_seconds"] = args["timeout_seconds"] + if cancellation_event is not None and _accepts_keyword(run_command, "cancel_event"): + kwargs["cancel_event"] = cancellation_event + if _accepts_keyword(run_command, "on_output"): + kwargs["on_output"] = self._command_progress(tool_call, args) + return run_command(**kwargs) + + def _command_progress(self, tool_call: ToolCall, args: dict[str, Any]) -> Callable[[str], None]: + """Stream the running command's latest output into its card.""" + metadata = _safe_tool_event_metadata("run_command", args, workspace_root=self._workspace_root) + title = metadata["title"] if metadata else "Run command" + + def on_output(tail: str) -> None: + self._event_sink.tool_call( + ToolCallEvent( + phase="progress", + tool_call_id=tool_call.tool_call_id, + tool_name=tool_call.name, + run_id="", + request_id="", + title=title, + kind="execute", + status="in_progress", + content_text=f"Running…\n{tail}", + ) + ) + + return on_output + + def _run_ask_user(self, tool_call, args, *, run_id, request_id, cancellation_event): + options = list(args.get("options") or []) + record = getattr(self._telemetry, "record", None) + if callable(record): + record( + event_type="agent.tool.completed", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "tool_name": "ask_user", + "tool_call_id": tool_call.tool_call_id, + "status": "completed", + "option_count": len(options), + "text": args["question"], + }, + ) + return { + "status": "ok", + "question": args["question"], + "options": options, + "note": "The question is shown to the user; their answer arrives as the next message.", + } + + def _run_update_plan(self, tool_call, args, *, run_id, request_id, cancellation_event): + entries: list[PlanEntrySpec] = list(args["entries"]) + emit_event( + self._event_sink, + "plan", + PlanEvent( + run_id=run_id, + request_id=request_id, + tool_call_id=tool_call.tool_call_id, + entries=tuple(entries), + ), + ) + record = getattr(self._telemetry, "record", None) + if callable(record): + record( + event_type="agent.tool.completed", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "tool_name": "update_plan", + "tool_call_id": tool_call.tool_call_id, + "status": "completed", + "entry_count": len(entries), + "entries": [asdict(entry) for entry in entries], + }, + ) + return { + "status": "ok", + "entry_count": len(entries), + "entries": [asdict(entry) for entry in entries], + } + + # ------------------------------------------------------------ cards + + def _emit_tool_start( + self, + tool_call: ToolCall, + arguments: dict[str, Any], + metadata: dict[str, Any] | None, + preview: EditPreview | None, + *, + pending: bool, + ) -> None: + if metadata is None: + return + self._event_sink.tool_call( + ToolCallEvent( + phase="started", + tool_call_id=tool_call.tool_call_id, + tool_name=tool_call.name, + run_id="", + request_id="", + title=metadata["title"], + kind=metadata["kind"], + status="pending" if pending else "in_progress", + path=metadata.get("path"), + diff_old_text=preview.old_text if preview is not None and preview.has_diff else None, + diff_new_text=preview.new_text if preview is not None and preview.has_diff else None, + raw_input=metadata.get("raw_input"), + locations=metadata.get("locations"), + extra_diffs=preview.extra_diffs if preview is not None else (), + ) + ) + + def _emit_tool_completed( + self, + tool_call: ToolCall, + arguments: dict[str, Any], + tool_output: dict[str, Any], + metadata: dict[str, Any] | None, + preview: EditPreview | None, + ) -> None: + if metadata is None: + return + has_diff = preview is not None and preview.has_diff + self._event_sink.tool_call( + ToolCallEvent( + phase="completed", + tool_call_id=tool_call.tool_call_id, + tool_name=tool_call.name, + run_id="", + request_id="", + title=metadata["title"], + kind=metadata["kind"], + status="completed", + path=metadata.get("path"), + diff_old_text=preview.old_text if has_diff else None, + diff_new_text=preview.new_text if has_diff else None, + content_text=_completed_card_text(tool_call.name, tool_output, metadata, has_diff), + raw_output=tool_output, + locations=metadata.get("locations"), + extra_diffs=preview.extra_diffs if preview is not None else (), + ) + ) + + def _emit_tool_failed( + self, + tool_call: ToolCall, + arguments: dict[str, Any], + metadata: dict[str, Any] | None, + preview: EditPreview | None, + *, + error: BaseException | None = None, + message: str | None = None, + ) -> None: + if metadata is None: + return + has_diff = preview is not None and preview.has_diff + text = message or f"{metadata['title']} failed: {error}" + self._event_sink.tool_call( + ToolCallEvent( + phase="failed", + tool_call_id=tool_call.tool_call_id, + tool_name=tool_call.name, + run_id="", + request_id="", + title=metadata["title"], + kind=metadata["kind"], + status="failed", + path=metadata.get("path"), + diff_old_text=preview.old_text if has_diff else None, + diff_new_text=preview.new_text if has_diff else None, + content_text=text, + locations=metadata.get("locations"), + extra_diffs=preview.extra_diffs if preview is not None else (), + ) + ) + + def _emit_denied( + self, + tool_call: ToolCall, + arguments: dict[str, Any], + metadata: dict[str, Any] | None, + *, + message: str, + ) -> None: + self._emit_tool_start(tool_call, arguments, metadata, None, pending=False) + self._emit_tool_failed(tool_call, arguments, metadata, None, message=f"Not run: {message}") + + def _emit_revision_card( + self, + tool_call: ToolCall, + arguments: dict[str, Any], + metadata: dict[str, Any] | None, + preview: EditPreview | None, + offer: RevisionOffer | None, + revision: _Revision, + ) -> None: + """Close the card of a revised call: the old→merged diff when kept hunks were written.""" + if revision.status != "applied": + detail = { + "file_changed": " (the file changed, so the kept parts were not applied)", + "write_failed": f" (applying the kept parts failed: {revision.error})", + }.get(revision.status, "") + self._emit_tool_failed( + tool_call, arguments, metadata, preview, message=f"Not run: revision requested{detail}." + ) + return + if metadata is None: + return + self._event_sink.tool_call( + ToolCallEvent( + phase="completed", + tool_call_id=tool_call.tool_call_id, + tool_name=tool_call.name, + run_id="", + request_id="", + title=metadata["title"], + kind=metadata["kind"], + status="completed", + path=metadata.get("path"), + diff_old_text=offer.old_text if offer is not None else None, + diff_new_text=revision.merged_text, + content_text=( + f"Applied {len(revision.kept)} of {revision.total} parts; " + "the agent is revising the rest." + ), + locations=metadata.get("locations"), + ) + ) + + # ---------------------------------------------------------- previews + + def _edit_preview( + self, + name: str, + arguments: dict[str, Any], + tool_call: ToolCall, + *, + run_id: str, + request_id: str, + ) -> EditPreview | None: + if name == "apply_patch": + return self._patch_preview( + arguments, tool_call_id=tool_call.tool_call_id, run_id=run_id, request_id=request_id + ) + if name not in {"create_file", "write_file", "replace_text", "edit_file", "delete_file"}: + return None + path = str(arguments.get("path", "")) + if name == "create_file": + return EditPreview(None, str(arguments.get("content", ""))) + try: + current = self._read_current_text(path, tool_call, run_id=run_id, request_id=request_id) + except (ToolFileNotFoundError, FileNotFoundError) as exc: + if name == "write_file": + current = "" + elif name == "delete_file": + return EditPreview(None, None) + else: + return EditPreview(None, None, exc) + except Exception as exc: # noqa: BLE001 + policy_denial = isinstance(exc, PermissionError) and getattr(exc, "errno", None) is None + if name == "delete_file" and not policy_denial: + # Directories, binaries, oversized or unreadable files: no diff, + # the handler decides (unlink needs only directory permissions). + return EditPreview(None, None) + # Policy denials were already recorded by read_text; fail before the + # handler would record them again. + return EditPreview(None, None, exc) + if name == "write_file": + return EditPreview(current, str(arguments.get("content", ""))) + if name == "delete_file": + return EditPreview(current, "") + if name == "replace_text": + edits = [ + TextEdit( + old_text=str(arguments.get("old_text", "")), + new_text=str(arguments.get("new_text", "")), + replace_all=bool(arguments.get("replace_all", False)), + ) + ] + else: + edits = list(arguments.get("edits", [])) + try: + new_text, _replacements = apply_text_edits(current, edits, path=path) + except ToolError as exc: + return EditPreview(current, None, exc) + return EditPreview(current, new_text) + + def _patch_preview( + self, arguments: dict[str, Any], *, tool_call_id: str, run_id: str, request_id: str + ) -> EditPreview | None: + plan = getattr(self._file_tools, "plan_patch", None) + if not callable(plan): + return None + try: + planned = plan( + arguments["_patch"], tool_call_id=tool_call_id, run_id=run_id, request_id=request_id + ) + except PatchError as exc: + return EditPreview(None, None, ToolError(str(exc), code="patch_does_not_apply")) + except Exception as exc: # noqa: BLE001 - outside workspace, binary file, ... + return EditPreview(None, None, exc) + diffs = [ + ( + _absolute_path(self._workspace_root, item.target) or item.target, + item.old_text, + item.new_text if item.new_text is not None else "", + ) + for item in planned + ] + if not diffs: + return None + _first_path, first_old, first_new = diffs[0] + return EditPreview(first_old, first_new, extra_diffs=tuple(diffs[1:])) + + def _read_current_text( + self, + path: str, + tool_call: ToolCall, + *, + run_id: str, + request_id: str, + ) -> str: + read_text = getattr(self._file_tools, "read_text", None) + if callable(read_text): + kwargs: dict[str, Any] = { + "strict": True, + "tool_call_id": tool_call.tool_call_id, + "run_id": run_id, + "request_id": request_id, + } + if _accepts_keyword(read_text, "tool_name"): + kwargs["tool_name"] = tool_call.name + value = read_text(path, **kwargs) + else: + value = self._file_tools.read_file( + path=path, + tool_call_id=tool_call.tool_call_id, + run_id=run_id, + request_id=request_id, + ) + return _coerce_text(value) + + +def _coerce_text(value: Any) -> str: + if isinstance(value, tuple) and value: + value = value[0] + if isinstance(value, str): + return value + content = getattr(value, "content", None) + if isinstance(content, str): + return content + if isinstance(value, dict) and isinstance(value.get("content"), str): + return value["content"] + return "" + + +def _accepts_keyword(function: Any, name: str) -> bool: + try: + parameters = inspect.signature(function).parameters + except (TypeError, ValueError): + return True + if name in parameters: + return True + return any(parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters.values()) + + +def _revision_telemetry( + decision: object, offer: RevisionOffer | None, revision: _Revision | None +) -> dict[str, Any]: + """Decision payload keys of a "Revise…" form; empty when no form was shown. + + The form outcome and status are behavioural and the counts system + metadata; the instructions travel as ``text`` (content: dropped when + content is not stored). + """ + action = getattr(decision, "elicitation_action", None) + if not isinstance(action, str): + return {} + payload: dict[str, Any] = { + "elicitation_action": action, + "hunk_count": len(offer.hunks) if offer is not None else 0, + } + if revision is not None: + payload["kept_hunk_count"] = len(revision.kept) + payload["revise_status"] = revision.status + if revision.instructions: + payload["text"] = revision.instructions + return payload + + +def _wrote_revision(result: dict[str, Any]) -> bool: + """A "Revise…" result whose kept hunks were written (the workspace changed).""" + return result.get("status") == "revise" and bool(result.get("applied_path")) + + +def _requires_manual_approval(tool_name: str) -> bool: + return requires_manual_approval(tool_name) + + +# -------------------------------------------------------- argument checks + + +def _arg_present(arguments: dict[str, Any], field_name: str) -> bool: + return field_name in arguments and arguments[field_name] is not None + + +def _require_str(arguments: dict[str, Any], field_name: str, tool: str) -> str: + value = arguments.get(field_name) + if not isinstance(value, str): + got = "nothing" if not _arg_present(arguments, field_name) else type(value).__name__ + raise ToolArgumentError( + f"Invalid argument '{field_name}' for {tool}: expected a string, got {got}.", + field=field_name, + ) + return value + + +def _optional_str(arguments: dict[str, Any], field_name: str, tool: str, default: str | None = None) -> str | None: + if not _arg_present(arguments, field_name): + return default + value = arguments[field_name] + if not isinstance(value, str): + raise ToolArgumentError( + f"Invalid argument '{field_name}' for {tool}: expected a string, got {type(value).__name__}.", + field=field_name, + ) + return value + + +def _arg_int( + arguments: dict[str, Any], + field_name: str, + tool: str, + *, + minimum: int | None = None, + maximum: int | None = None, + default: int | None = None, + adjustments: list[str] | None = None, +) -> int | None: + """Parse an integer argument. + + With ``adjustments``, an out-of-range value is clamped to the bound and a + note is recorded for the tool result (size and paging arguments, where a + clamp is always what the model meant); without it, out of range is an error. + """ + if not _arg_present(arguments, field_name): + return default + value = arguments[field_name] + parsed: int | None = None + if isinstance(value, bool): + parsed = None + elif isinstance(value, int): + parsed = value + elif isinstance(value, float) and value.is_integer(): + parsed = int(value) + elif isinstance(value, str) and value.strip().lstrip("-").isdigit(): + parsed = int(value.strip()) + if parsed is None: + raise ToolArgumentError( + f"Invalid argument '{field_name}' for {tool}: expected an integer, got {value!r}.", + field=field_name, + ) + if adjustments is not None and minimum is not None and parsed < minimum: + adjustments.append(f"{field_name} {parsed} was raised to the minimum {minimum}") + return minimum + if adjustments is not None and maximum is not None and parsed > maximum: + adjustments.append(f"{field_name} {parsed} was lowered to the maximum {maximum}") + return maximum + if (minimum is not None and parsed < minimum) or (maximum is not None and parsed > maximum): + bounds = [] + if minimum is not None: + bounds.append(f">= {minimum}") + if maximum is not None: + bounds.append(f"<= {maximum}") + raise ToolArgumentError( + f"Invalid argument '{field_name}' for {tool}: expected an integer {' and '.join(bounds)}, got {parsed}.", + field=field_name, + ) + return parsed + + +def _arg_number(arguments: dict[str, Any], field_name: str, tool: str) -> float | None: + if not _arg_present(arguments, field_name): + return None + value = arguments[field_name] + if isinstance(value, bool): + pass + elif isinstance(value, (int, float)): + return float(value) + elif isinstance(value, str): + try: + return float(value.strip()) + except ValueError: + pass + raise ToolArgumentError( + f"Invalid argument '{field_name}' for {tool}: expected a number, got {value!r}.", + field=field_name, + ) + + +def _arg_bool(arguments: dict[str, Any], field_name: str, tool: str, default: bool = False) -> bool: + if not _arg_present(arguments, field_name): + return default + value = arguments[field_name] + if isinstance(value, bool): + return value + if isinstance(value, str) and value.strip().lower() in {"true", "false"}: + return value.strip().lower() == "true" + raise ToolArgumentError( + f"Invalid argument '{field_name}' for {tool}: expected true or false, got {value!r}.", + field=field_name, + ) + + +def _arg_enum( + arguments: dict[str, Any], + field_name: str, + tool: str, + choices: Sequence[str], + default: str, +) -> str: + if not _arg_present(arguments, field_name): + return default + value = arguments[field_name] + if not isinstance(value, str) or value not in choices: + raise ToolArgumentError( + f"Invalid argument '{field_name}' for {tool}: expected one of {', '.join(choices)}, got {value!r}.", + field=field_name, + ) + return value + + +# Characters that would need a shell; such a string is never split into argv. +_SHELL_SYNTAX = frozenset("|&;<>()$`*?[]{}~!#\n\\") + + +def _require_str_list( + arguments: dict[str, Any], field_name: str, tool: str, *, adjustments: list[str] | None = None +) -> list[str]: + value = arguments.get(field_name) + if isinstance(value, str): + if adjustments is not None and value.strip() and not (_SHELL_SYNTAX & set(value)): + try: + split = shlex.split(value) + except ValueError: + split = [] + if split: + adjustments.append(f"{field_name} was given as one string and split into {split!r}") + return split + raise ToolArgumentError( + f"Invalid argument '{field_name}' for {tool}: expected a list of strings, got a single " + "string. Split the command into separate argv items; argv is not passed through a " + "shell, so pipes, redirection and globs need a shell program as argv[0] (for example " + '["bash", "-c", "..."]).', + field=field_name, + ) + if not isinstance(value, (list, tuple)) or not value or not all(isinstance(item, str) for item in value): + raise ToolArgumentError( + f"Invalid argument '{field_name}' for {tool}: expected a non-empty list of strings.", + field=field_name, + ) + return [str(item) for item in value] + + +def _require_edits(arguments: dict[str, Any], tool: str) -> list[TextEdit]: + value = arguments.get("edits") + if not isinstance(value, list) or not value: + raise ToolArgumentError( + f"Invalid argument 'edits' for {tool}: expected a non-empty list of " + "{{old_text, new_text}} objects.", + field="edits", + ) + if len(value) > 50: + raise ToolArgumentError( + f"Invalid argument 'edits' for {tool}: at most 50 edits per call.", field="edits" + ) + edits: list[TextEdit] = [] + for index, item in enumerate(value): + if not isinstance(item, dict): + raise ToolArgumentError( + f"Invalid argument 'edits[{index}]' for {tool}: expected an object.", + field=f"edits[{index}]", + ) + for key in ("old_text", "new_text"): + if not isinstance(item.get(key), str): + raise ToolArgumentError( + f"Invalid argument 'edits[{index}].{key}' for {tool}: expected a string.", + field=f"edits[{index}].{key}", + ) + try: + replace_all = _arg_bool(item, "replace_all", tool) + except ToolArgumentError as exc: + raise ToolArgumentError(str(exc), field=f"edits[{index}].{exc.field}") from None + edits.append( + TextEdit(old_text=item["old_text"], new_text=item["new_text"], replace_all=replace_all) + ) + return edits + + +def _require_plan_entries(arguments: dict[str, Any], tool: str) -> list[PlanEntrySpec]: + value = arguments.get("entries") + if not isinstance(value, list) or not value: + raise ToolArgumentError( + f"Invalid argument 'entries' for {tool}: expected a non-empty list of plan entries.", + field="entries", + ) + if len(value) > 30: + raise ToolArgumentError( + f"Invalid argument 'entries' for {tool}: at most 30 entries.", field="entries" + ) + entries: list[PlanEntrySpec] = [] + for index, item in enumerate(value): + if not isinstance(item, dict): + raise ToolArgumentError( + f"Invalid argument 'entries[{index}]' for {tool}: expected an object.", + field=f"entries[{index}]", + ) + content = item.get("content") + if not isinstance(content, str) or not content.strip(): + raise ToolArgumentError( + f"Invalid argument 'entries[{index}].content' for {tool}: expected a non-empty string.", + field=f"entries[{index}].content", + ) + if not _arg_present(item, "status"): + raise ToolArgumentError( + f"Invalid argument 'entries[{index}].status' for {tool}: status is required.", + field=f"entries[{index}].status", + ) + try: + status = _arg_enum(item, "status", tool, ("pending", "in_progress", "completed"), "pending") + priority = _arg_enum(item, "priority", tool, ("high", "medium", "low"), "medium") + except ToolArgumentError as exc: + raise ToolArgumentError(str(exc), field=f"entries[{index}].{exc.field}") from None + entries.append(PlanEntrySpec(content=content.strip(), status=status, priority=priority)) + return entries + + +_SHELL_PROGRAMS = ("bash", "sh", "zsh") +_SHELL_LIKE_TOOL_NAMES = frozenset({"bash", "sh", "shell", "terminal", "exec", "execute", "execute_command", + "run_shell", "run_terminal_cmd", "cmd", "powershell"}) +_NO_SHELL_SENTENCE = ( + "There is no shell: pipes, globs, '&&', 'cd' and redirection are not available; pass " + "arguments as separate argv items." +) + + +def _usable_shell(blocked: object) -> str | None: + """The first installed shell the study does not block, else None.""" + if not isinstance(blocked, (set, frozenset, list, tuple)): + return None + from code4me2_agent.command_tools import available_commands, command_key + + keys = {command_key(str(name)) for name in blocked} + installed = available_commands([shell for shell in _SHELL_PROGRAMS if shell not in keys]) + return installed[0] if installed else None + + +def _shell_sentence(shell: str) -> str: + return ( + "argv is not passed through a shell; for pipes, globs, '&&', 'cd' or redirection run a " + f'shell explicitly, e.g. ["{shell}", "-c", "cd src && pytest -q | tail -20"].' + ) + + +def _with_shell_description(definition: dict[str, Any], shell: str) -> dict[str, Any]: + function = definition.get("function") or {} + description = function.get("description") + if function.get("name") != "run_command" or not isinstance(description, str): + return definition + if _NO_SHELL_SENTENCE not in description: + return definition + return { + **definition, + "function": {**function, "description": description.replace(_NO_SHELL_SENTENCE, _shell_sentence(shell))}, + } + + +def _validate_arguments(name: str, arguments: dict[str, Any]) -> dict[str, Any]: + """Return a validated, normalized argument mapping for a catalogue tool. + + Unknown (MCP or unsupported) tools pass through unchanged. Size and paging + arguments out of range are clamped, and an argv given as one plain string is + split; each adjustment is reported to the model as ``argument_notes``. + """ + adjustments: list[str] = [] + validated = _validate_arguments_strict(name, arguments, adjustments) + if adjustments: + validated = {**validated, "_argument_notes": adjustments} + return validated + + +def _validate_arguments_strict( + name: str, arguments: dict[str, Any], adjustments: list[str] +) -> dict[str, Any]: + if name == "read_file": + return { + "path": _require_str(arguments, "path", name), + "offset": _arg_int(arguments, "offset", name, minimum=1, adjustments=adjustments) + or _arg_int(arguments, "line_start", name, minimum=1, adjustments=adjustments), + "limit": _arg_int(arguments, "limit", name, minimum=1, maximum=5000, adjustments=adjustments) + or _line_end_limit(arguments, name), + } + if name == "create_file": + return { + "path": _require_str(arguments, "path", name), + "content": _require_str(arguments, "content", name), + } + if name == "write_file": + return { + "path": _require_str(arguments, "path", name), + "content": _require_str(arguments, "content", name), + "force": _arg_bool(arguments, "force", name), + } + if name == "replace_text": + return { + "path": _require_str(arguments, "path", name), + "old_text": _require_str(arguments, "old_text", name), + "new_text": _require_str(arguments, "new_text", name), + "replace_all": _arg_bool(arguments, "replace_all", name), + "force": _arg_bool(arguments, "force", name), + } + if name == "edit_file": + return { + "path": _require_str(arguments, "path", name), + "edits": _require_edits(arguments, name), + "force": _arg_bool(arguments, "force", name), + } + if name == "apply_patch": + patch = arguments.get("patch", arguments.get("input")) + if not isinstance(patch, str) or not patch.strip(): + raise ToolArgumentError( + "Invalid argument 'patch' for apply_patch: expected the patch text, starting with " + "'*** Begin Patch' and ending with '*** End Patch'.", + field="patch", + ) + return {"patch": patch, "force": _arg_bool(arguments, "force", name)} + if name == "ask_user": + question = _require_str(arguments, "question", name).strip() + if not question: + raise ToolArgumentError( + "Invalid argument 'question' for ask_user: expected a non-empty question.", + field="question", + ) + options = arguments.get("options") + if options is None: + options = [] + if not isinstance(options, list) or len(options) > 6 or not all( + isinstance(option, str) and option.strip() and len(option) <= 200 for option in options + ): + raise ToolArgumentError( + "Invalid argument 'options' for ask_user: expected at most 6 short answer choices.", + field="options", + ) + return {"question": question[:2000], "options": [option.strip() for option in options]} + if name == "delete_file": + return {"path": _require_str(arguments, "path", name)} + if name == "move_file": + return { + "source_path": _require_str(arguments, "source_path", name), + "destination_path": _require_str(arguments, "destination_path", name), + "overwrite": _arg_bool(arguments, "overwrite", name), + } + if name == "list_files": + return { + "path": _optional_str(arguments, "path", name, ".") or ".", + "recursive": _arg_bool(arguments, "recursive", name), + "max_depth": _arg_int(arguments, "max_depth", name, minimum=1, maximum=20, adjustments=adjustments), + "max_results": _arg_int( + arguments, "max_results", name, minimum=1, maximum=2000, default=500, adjustments=adjustments + ), + "include_ignored": _arg_bool(arguments, "include_ignored", name), + } + if name == "glob_files": + return { + "pattern": _require_str(arguments, "pattern", name), + "path": _optional_str(arguments, "path", name, ".") or ".", + "max_results": _arg_int( + arguments, "max_results", name, minimum=1, maximum=2000, default=500, adjustments=adjustments + ), + "include_ignored": _arg_bool(arguments, "include_ignored", name), + } + if name == "grep_files": + return { + "pattern": _require_str(arguments, "pattern", name), + "path": _optional_str(arguments, "path", name, ".") or ".", + "glob": _optional_str(arguments, "glob", name), + "case_insensitive": _arg_bool(arguments, "case_insensitive", name), + "context_lines": _arg_int( + arguments, "context_lines", name, minimum=0, maximum=10, default=0, adjustments=adjustments + ), + "max_results": _arg_int( + arguments, "max_results", name, minimum=1, maximum=1000, default=200, adjustments=adjustments + ), + "output_mode": _arg_enum( + arguments, "output_mode", name, ("content", "files_with_matches", "count"), "content" + ), + "include_ignored": _arg_bool(arguments, "include_ignored", name), + } + if name == "search_files": + return { + "query": _require_str(arguments, "query", name), + "path": _optional_str(arguments, "path", name, ".") or ".", + } + if name == "run_command": + return { + "argv": _require_str_list(arguments, "argv", name, adjustments=adjustments), + "cwd": _optional_str(arguments, "cwd", name, ".") or ".", + "timeout_seconds": _arg_number(arguments, "timeout_seconds", name), + } + if name == "update_plan": + return {"entries": _require_plan_entries(arguments, name)} + return dict(arguments) + + +def _line_end_limit(arguments: dict[str, Any], tool: str) -> int | None: + line_end = _arg_int(arguments, "line_end", tool, minimum=1) + if line_end is None: + return None + start = _arg_int(arguments, "line_start", tool, minimum=1) or 1 + return max(1, min(line_end - start + 1, 5000)) + + +# ---------------------------------------------------------------- providers + + +class FakeOpenAICompatibleProvider: + """Scripted provider for tests: one script entry per model call.""" + + def __init__(self, script: list[dict[str, object]]) -> None: + self._script = [dict(item) for item in script] + self.calls: list[dict[str, Any]] = [] + + def generate( + self, + messages: list[dict[str, Any]], + *, + run_id: str = "", + tool_choice: str | None = "auto", + cancellation_event: Event | None = None, + include_tools: bool = True, + ) -> ProviderTurn: + self.calls.append( + { + "messages": [dict(message) for message in messages], + "tool_choice": tool_choice, + "include_tools": include_tools, + } + ) + if not self._script: + raise FakeProviderExhaustedError("The fake provider script is exhausted.") + step = self._script.pop(0) + if "delay_seconds" in step: + _sleep_interruptible(float(step["delay_seconds"]), cancellation_event, sleep) + output: dict[str, Any] = {"tool_calls": list(step.get("tool_calls") or [])} + text = step.get("final_answer", step.get("text")) + if text is not None: + output["final_answer"] = str(text) + reasoning = _normalize_optional_text(step.get("reasoning")) + if reasoning is not None: + output["reasoning"] = reasoning + if isinstance(step.get("reasoning_fields"), dict): + output["reasoning_fields"] = dict(step["reasoning_fields"]) + raw_usage = step.get("usage") + usage = _normalize_usage(raw_usage, messages, output) + return ProviderTurn( + output=output, + usage=usage, + finish_reason=_normalize_optional_text(step.get("finish_reason")), + model=str(step.get("model") or "fake-model"), + request_payload={"messages": messages, "tool_choice": tool_choice}, + raw_response=dict(step), + usage_estimated=not isinstance(raw_usage, dict), + ) + + +class OpenAICompatibleProvider: + def __init__( + self, + *, + kind: str, + base_url: str, + model: str, + api_key_env: str, + timeout_seconds: float, + auth_headers: dict[str, str] | None = None, + tool_definitions: list[dict[str, Any]] | None = None, + temperature: float | None = None, + session_id: str = "", + managed_request: Any | None = None, + max_output_tokens: int | None = None, + retry_policy: RetryPolicy | None = None, + request_deadline_seconds: float | None = None, + extra_body: dict[str, Any] | None = None, + ) -> None: + self._kind = kind.strip() or "code4me_backend" + self._request_deadline_seconds = request_deadline_seconds + # Only direct calls take extra fields; the backend relays forward a fixed shape. + self._extra_body = dict(extra_body or {}) if self._kind == "openai" else {} + self._base_url = base_url.rstrip("/") + self._model = model + self._api_key_env = api_key_env + self._timeout_seconds = timeout_seconds + self._auth_headers = dict(auth_headers or {}) + # An empty list is an explicit server-assigned policy: do not replace it + # with the local default tools, which would make the backend reject the + # managed request as broader than the frozen profile. + self._tool_definitions = ( + tool_catalog.tool_definitions() if tool_definitions is None else list(tool_definitions) + ) + self._temperature = temperature + self._session_id = session_id + self._managed_request = managed_request + self._max_output_tokens = max_output_tokens + self._retry_policy = retry_policy or RetryPolicy.from_env() + self._client_instance: OpenAI | None = None + # Reasoning pass-back: echo what the model produced, and fill "" where a + # DeepSeek V4 model requires the field. Each adapts at most once when + # the server says otherwise (see generate()). + self._echo_reasoning = True + self._fill_reasoning = _needs_reasoning_on_every_assistant(model) + + @property + def retry_policy(self) -> RetryPolicy: + return self._retry_policy + + def generate( + self, + messages: list[dict[str, Any]], + *, + run_id: str = "", + tool_choice: str | None = "auto", + cancellation_event: Event | None = None, + include_tools: bool = True, + ) -> ProviderTurn: + request_payload: dict[str, Any] = { + "model": self._model, + "messages": [_to_openai_message(message) for message in messages], + } + if self._tool_definitions and include_tools: + request_payload["tools"] = list(self._tool_definitions) + request_payload["tool_choice"] = tool_choice or "auto" + _apply_reasoning_policy( + request_payload["messages"], + echo=self._echo_reasoning, + fill=self._fill_reasoning and "tools" in request_payload, + ) + if self._temperature is not None: + request_payload["temperature"] = self._temperature + if self._max_output_tokens: + request_payload["max_tokens"] = int(self._max_output_tokens) + # As sent on the wire (and so as recorded in telemetry): top-level fields. + for key, value in self._extra_body.items(): + request_payload.setdefault(key, value) + + if self._kind == "managed_backend": + if not callable(self._managed_request): + raise ProviderRequestFailed("Managed backend transport is unavailable.") + response_payload = _call_with_retries( + lambda: self._managed_request( + run_id=run_id, + session_id=self._session_id, + model_request=request_payload, + ), + policy=self._retry_policy, + classify=_classify_managed_error(self._retry_policy), + cancellation_event=cancellation_event, + ) + else: + def call() -> Any: + return _call_with_retries( + lambda: self._sdk_call(request_payload, cancellation_event), + policy=self._retry_policy, + classify=_classify_sdk_error(self._retry_policy), + cancellation_event=cancellation_event, + before_attempt=_rate_limit_provider_request_from_env, + ) + + try: + response_payload = call() + except ProviderRequestFailed as exc: + # The server said how it treats reasoning on assistant messages: + # adapt once for this session and resend the same request. + verdict = _reasoning_rejection(exc) + if verdict == "rejected" and self._echo_reasoning: + self._echo_reasoning = False + elif verdict == "required" and not self._fill_reasoning: + self._fill_reasoning = True + else: + raise + logging.warning("Model service %s reasoning pass-back; adapting: %s", verdict, exc) + _apply_reasoning_policy( + request_payload["messages"], + echo=self._echo_reasoning, + fill=self._fill_reasoning and "tools" in request_payload, + ) + response_payload = call() + if not isinstance(response_payload, dict): + raise ProviderRequestFailed("The model service returned a non-object response.") + normalized_output = _normalize_openai_provider_response(response_payload) + raw_usage = response_payload.get("usage") + return ProviderTurn( + output=normalized_output, + usage=_normalize_usage(raw_usage, messages, normalized_output), + finish_reason=_response_finish_reason(response_payload), + model=str(response_payload.get("model") or self._model), + request_payload=request_payload, + raw_response=response_payload, + usage_estimated=not _usage_is_reported(raw_usage), + ) + + def _sdk_call( + self, + request_payload: dict[str, Any], + cancellation_event: Event | None, + ) -> dict[str, Any]: + client = self._client() + logging.info( + "Agent model request via OpenAI SDK: kind=%s endpoint=%s model=%s", + self._kind, + f"{self._openai_base_url()}/chat/completions", + self._model, + ) + # The SDK refuses unknown keyword arguments; extra fields go in extra_body. + arguments = dict(request_payload) + extra = {key: arguments.pop(key) for key in self._extra_body if key in arguments} + if extra: + arguments["extra_body"] = extra + # A daemon thread (not an executor) so an abandoned request after a + # cancel can never delay interpreter exit when stdin closes. + outcome: dict[str, Any] = {} + finished = Event() + + def worker() -> None: + try: + outcome["value"] = client.chat.completions.with_raw_response.create(**arguments) + except BaseException as exc: # noqa: BLE001 + outcome["error"] = exc + finally: + finished.set() + + Thread(target=worker, daemon=True, name="code4me2-provider").start() + deadline = ( + perf_counter() + self._request_deadline_seconds + if self._request_deadline_seconds + else None + ) + while not finished.wait(0.25): + if _turn_was_cancelled(cancellation_event): + raise ProviderCancelled("Cancelled while waiting for the model response.") + if deadline is not None and perf_counter() > deadline: + # Retryable like any timeout; the abandoned daemon thread is dropped. + raise APITimeoutError( + request=httpx.Request("POST", f"{self._openai_base_url()}/chat/completions") + ) + if "error" in outcome: + raise outcome["error"] + raw_response = outcome["value"] + logging.info( + "Agent model response via OpenAI SDK: status=%s model=%s", + raw_response.http_response.status_code, + self._model, + ) + payload = raw_response.http_response.json() + return payload if isinstance(payload, dict) else {"choices": payload} + + def _client(self) -> OpenAI: + if self._client_instance is None: + self._client_instance = _OpenAIClientWithoutSdkAuth( + api_key="code4me-local-client", + base_url=self._openai_base_url(), + timeout=self._timeout_seconds, + default_headers=self._headers(), + max_retries=0, + ) + return self._client_instance + + def _headers(self) -> dict[str, str]: + headers = {"User-Agent": USER_AGENT} + api_key = os.getenv(self._api_key_env, "").strip() if self._api_key_env else "" + if self._kind == "code4me_backend": + headers.update(self._auth_headers) + elif self._kind == "openai": + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + headers.update(_opencode_session_headers(self._base_url, self._session_id)) + elif self._kind != "managed_backend": + raise ValueError(f"Unsupported provider kind: {self._kind}") + return headers + + def _openai_base_url(self) -> str: + if self._kind == "code4me_backend": + return f"{self._base_url}/api/acp" + base_url = self._base_url + if not base_url.endswith("/v1"): + base_url = f"{base_url}/v1" + return base_url + + +# OpenCode (Zen and Go) rejects a request without ``x-opencode-session`` +# (400 MissingSessionID). Same scheme as the backend relay (agents/provider.py): +# a one-way hash that stays stable for one agent session. +_OPENCODE_SESSION_DOMAIN = b"code4me-opencode-session\x00" + + +def _opencode_session_headers(base_url: str, session_id: str) -> dict[str, str]: + host = (urlparse(base_url).hostname or "").lower() + if not (host == "opencode.ai" or host.endswith(".opencode.ai")): + return {} + digest = hashlib.sha256(_OPENCODE_SESSION_DOMAIN + (session_id or "").encode("utf-8")) + return {"x-opencode-session": f"c4m-{digest.hexdigest()[:32]}"} + + +class _OpenAIClientWithoutSdkAuth(OpenAI): + @property + def auth_headers(self) -> dict[str, str]: + return {} + + +# ------------------------------------------------------------------- memory + + +def _estimate_text_tokens(text: str) -> int: + return len(text) // 4 + 1 + + +def _estimate_tokens(message: dict[str, Any]) -> int: + try: + serialized = json.dumps(message, ensure_ascii=False, default=str) + except (TypeError, ValueError): + serialized = str(message) + return len(serialized) // 4 + 4 + + +def _unit_tokens(unit: list[dict[str, Any]]) -> int: + return sum(_estimate_tokens(message) for message in unit) + + +def _split_units(messages: list[dict[str, Any]]) -> list[list[dict[str, Any]]]: + """Group messages so an assistant tool-call message stays with its results.""" + units: list[list[dict[str, Any]]] = [] + index = 0 + while index < len(messages): + message = messages[index] + role = message.get("role") + if role == "assistant" and message.get("tool_calls"): + unit = [message] + index += 1 + while index < len(messages) and messages[index].get("role") == "tool": + unit.append(messages[index]) + index += 1 + units.append(unit) + continue + if role == "tool": + # Stray result without its call; dropping it keeps the request valid. + index += 1 + continue + units.append([message]) + index += 1 + return units + + +_ELIDE_MIN_CHARS = 600 +_ELIDE_NOTE = ( + "Earlier tool output removed from context to save space; call the tool again if you need it." +) + + +def _elide_tool_message(message: dict[str, Any]) -> dict[str, Any]: + content = message.get("content") + if not isinstance(content, str) or len(content) <= _ELIDE_MIN_CHARS: + return message + status = "ok" + try: + parsed = json.loads(content) + if isinstance(parsed, dict): + if parsed.get("elided"): + return message + status = str(parsed.get("status", "ok")) + except (json.JSONDecodeError, TypeError): + pass + elided = dict(message) + elided["content"] = json.dumps( + {"status": status, "elided": True, "note": _ELIDE_NOTE, "preview": content[:160]}, + sort_keys=True, + ) + return elided + + +def _elide_unit(unit: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + _elide_tool_message(message) if message.get("role") == "tool" else message + for message in unit + ] + + +def _repair_tool_groups(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Insert missing tool results and drop orphaned ones from a persisted history.""" + repaired: list[dict[str, Any]] = [] + index = 0 + while index < len(messages): + message = messages[index] + role = message.get("role") + if role == "assistant" and message.get("tool_calls"): + repaired.append(message) + expected: dict[str, str] = {} + for call in message.get("tool_calls") or []: + if isinstance(call, dict) and call.get("id") is not None: + expected[str(call["id"])] = str(call.get("name", "")) + index += 1 + seen: set[str] = set() + while index < len(messages) and messages[index].get("role") == "tool": + result = messages[index] + call_id = str(result.get("tool_call_id", "")) + if call_id in expected and call_id not in seen: + repaired.append(result) + seen.add(call_id) + index += 1 + for call_id, name in expected.items(): + if call_id not in seen: + repaired.append( + { + "role": "tool", + "tool_call_id": call_id, + "name": name, + "content": json.dumps( + { + "status": "cancelled", + "error": "Result unavailable (interrupted before completion).", + "tool_name": name, + "tool_call_id": call_id, + }, + sort_keys=True, + ), + } + ) + continue + if role == "tool": + index += 1 + continue + repaired.append(message) + index += 1 + return repaired + + +class MemoryWindow: + def __init__(self, *, strategy: str, max_messages: int, max_tokens: int) -> None: + self._strategy = ( + strategy + if strategy in {"last_messages", "token_window"} + else "last_messages" + ) + self._max_messages = max(1, max_messages) + self._max_tokens = max(1, max_tokens) + self._messages: list[dict[str, Any]] = [] + # Provider tokens per estimated (chars/4) token, learned from usage. + # Code, line numbers and escaped JSON tokenize denser than chars/4. + self._scale = 1.0 + + @property + def max_tokens(self) -> int: + return self._max_tokens + + @property + def scale(self) -> float: + return self._scale + + def calibrate(self, *, provider_prompt_tokens: int, estimated_tokens: int) -> None: + """Adopt the provider's own count of the last request. + + Never below 1.0 (the estimate stays a floor) and capped so one odd + response cannot collapse the window. + """ + if provider_prompt_tokens > 0 and estimated_tokens > 0: + self._scale = min(_MAX_TOKEN_SCALE, max(1.0, provider_prompt_tokens / estimated_tokens)) + + def compact_current_turn( + self, *, reserve_tokens: int = 0, low_water: float = _CURRENT_TURN_LOW_WATER + ) -> dict[str, int] | None: + """Elide old tool output of the current turn in one step, in place. + + Trimming just enough on every call moves the cut forward each time and + so changes the request prefix on every call, which defeats provider + prompt caching. Eliding down to ``low_water`` of the budget once keeps + the prefix stable until the turn has grown back to the budget. + Returns what was done, or None when the turn still fits. + """ + system, budget, older, protected = self._partition(reserve_tokens) + used = sum(_unit_tokens(unit) for unit in protected) + # Same trigger as _shrink_protected: earlier turns are window()'s and + # summarisation's to drop; only a turn that alone overflows is touched. + if used <= budget or len(protected) <= 2: + return None + target = int(budget * low_water) + shrunk = list(protected) + elided = 0 + # Never the user prompt (0) or the newest unit, which the model is acting on. + for position in range(1, len(shrunk) - 1): + replacement = _elide_unit(shrunk[position]) + if replacement != shrunk[position]: + shrunk[position] = replacement + elided += 1 + if sum(_unit_tokens(unit) for unit in shrunk) <= target: + break + if not elided: + return None + rebuilt: list[dict[str, Any]] = [system] if system is not None else [] + for unit in [*older, *shrunk]: + rebuilt.extend(unit) + self._messages = rebuilt + return { + "elided_units": elided, + "estimated_tokens_before": used, + "estimated_tokens_after": sum(_unit_tokens(unit) for unit in shrunk), + "budget_tokens": budget, + } + + def append(self, message: dict[str, Any]) -> None: + self._messages.append(dict(message)) + self._bound_history() + + def replace_messages(self, messages: list[dict[str, Any]]) -> None: + cleaned = [dict(message) for message in messages if isinstance(message, dict)] + system, rest = _split_system(cleaned) + repaired = _repair_tool_groups(rest) + self._messages = ([system] if system is not None else []) + repaired + + def snapshot(self) -> list[dict[str, Any]]: + return [dict(message) for message in self._messages] + + def message_count(self) -> int: + return len(self._messages) + + def older_unit_count(self) -> int: + _system, _budget, older, _protected = self._partition(0) + return len(older) + + def clear(self) -> None: + system, _rest = self._split_pinned_system() + self._messages = [system] if system is not None else [] + + def ensure_system_message(self, content: str) -> None: + message = {"role": "system", "content": content} + if self._messages and self._messages[0].get("role") == "system": + self._messages[0] = message + return + self._messages.insert(0, message) + + def estimated_tokens(self) -> int: + return sum(_estimate_tokens(message) for message in self._messages) + + def _partition( + self, reserve_tokens: int + ) -> tuple[dict[str, Any] | None, int, list[list[dict[str, Any]]], list[list[dict[str, Any]]]]: + """``(system, budget, older units, protected units of the current turn)``.""" + system, rest = self._split_pinned_system() + budget = int(self._max_tokens / self._scale) - max(0, int(reserve_tokens)) + if system is not None: + budget -= _estimate_tokens(system) + units = _split_units(rest) + last_user = -1 + for position, unit in enumerate(units): + # Runtime-authored notes (review findings, checkpoints) are not the + # user's request: the turn starts at the user's own message. + if unit[0].get("role") == "user" and not _is_runtime_message(unit[0]): + last_user = position + if last_user < 0: + return system, budget, units, [] + return system, budget, units[:last_user], units[last_user:] + + def compaction_plan(self, *, reserve_tokens: int = 0, keep_recent_units: int = 4) -> int: + """How many of the oldest units to summarise, or 0 when everything still fits. + + Summarisation is the second layer: it is only needed once whole units + would be dropped even with their tool output elided. The most recent + ``keep_recent_units`` older units stay verbatim. + """ + _system, budget, older, protected = self._partition(reserve_tokens) + if len(older) <= keep_recent_units: + return 0 + used = sum(_unit_tokens(unit) for unit in protected) + fits_elided = used + sum(_unit_tokens(_elide_unit(unit)) for unit in older) <= budget + if fits_elided: + return 0 + count = len(older) - keep_recent_units + if count == 1 and _is_runtime_message(older[0][0]): + return 0 # only the previous checkpoint would be re-summarised + return count + + def oldest_units(self, count: int) -> list[list[dict[str, Any]]]: + _system, _budget, older, _protected = self._partition(0) + return [list(unit) for unit in older[:count]] + + def compact(self, count: int, summary: str) -> None: + """Replace the ``count`` oldest units with one checkpoint message.""" + system, rest = self._split_pinned_system() + units = _split_units(rest) + _system, _budget, older, _protected = self._partition(0) + count = max(0, min(count, len(older))) + if count == 0: + return + checkpoint = { + "role": "user", + "content": f"{CHECKPOINT_PREFIX}\n{summary.strip()}", + "code4me_runtime": "checkpoint", + } + rebuilt: list[dict[str, Any]] = [system] if system is not None else [] + rebuilt.append(checkpoint) + for unit in units[count:]: + rebuilt.extend(unit) + self._messages = rebuilt + + def window(self, *, reserve_tokens: int = 0) -> list[dict[str, Any]]: + system, budget, older, protected = self._partition(reserve_tokens) + + message_budget = self._max_messages if self._strategy == "last_messages" else None + used = sum(_unit_tokens(unit) for unit in protected) + message_count = sum(len(unit) for unit in protected) + + chosen: list[list[dict[str, Any]]] = [] + position = len(older) - 1 + # Pass 1: full units, newest first, stopping at the first that does not fit. + while position >= 0: + unit = older[position] + cost = _unit_tokens(unit) + if used + cost > budget or ( + message_budget is not None and message_count + len(unit) > message_budget + ): + break + chosen.append(unit) + used += cost + message_count += len(unit) + position -= 1 + # Pass 2: the remaining older units with their tool output elided. + while position >= 0: + unit = _elide_unit(older[position]) + cost = _unit_tokens(unit) + if used + cost > budget or ( + message_budget is not None and message_count + len(unit) > message_budget + ): + break + chosen.append(unit) + used += cost + message_count += len(unit) + position -= 1 + chosen.reverse() + + protected = _shrink_protected(protected, budget) + + selected: list[dict[str, Any]] = [] + if system is not None: + selected.append(system) + for unit in chosen: + selected.extend(unit) + for unit in protected: + selected.extend(unit) + return [dict(message) for message in selected] + + def _split_pinned_system( + self, + ) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]: + return _split_system(self._messages) + + def _bound_history(self) -> None: + """Keep the persisted history bounded for long sessions.""" + total = self.estimated_tokens() + if total <= 4 * self._max_tokens: + return + system, rest = self._split_pinned_system() + units = _split_units(rest) + keep_full = 6 + if len(units) > keep_full: + units = [_elide_unit(unit) for unit in units[:-keep_full]] + units[-keep_full:] + while units and sum(_unit_tokens(unit) for unit in units) > 6 * self._max_tokens: + if len(units) <= 1: + break + units.pop(0) + rebuilt: list[dict[str, Any]] = [] + if system is not None: + rebuilt.append(system) + for unit in units: + rebuilt.extend(unit) + self._messages = rebuilt + + +CHECKPOINT_PREFIX = ( + "[Context checkpoint written by the Code4Me runtime: a summary of the earlier part of this " + "session, which no longer fits the context window. It is not a new request.]" +) + + +def _is_runtime_message(message: dict[str, Any]) -> bool: + return bool(message.get("code4me_runtime")) + + +def _split_system( + messages: list[dict[str, Any]], +) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]: + if messages and messages[0].get("role") == "system": + return dict(messages[0]), messages[1:] + return None, list(messages) + + +def _shrink_protected( + protected: list[list[dict[str, Any]]], budget: int +) -> list[list[dict[str, Any]]]: + """Elide older tool groups of the current turn when it alone exceeds the budget.""" + if not protected or sum(_unit_tokens(unit) for unit in protected) <= budget: + return protected + shrunk = list(protected) + # Never touch the user prompt (index 0); elide older units first. + for position in range(1, len(shrunk) - 1): + shrunk[position] = _elide_unit(shrunk[position]) + if sum(_unit_tokens(unit) for unit in shrunk) <= budget: + break + if sum(_unit_tokens(unit) for unit in shrunk) > budget and len(shrunk) > 1: + # The newest unit itself is oversize (a batch of large tool results): + # elide its results oldest-first, keeping the most recent one intact, + # and only then everything. + newest = list(shrunk[-1]) + tool_positions = [i for i, message in enumerate(newest) if message.get("role") == "tool"] + for position in tool_positions[:-1]: + newest[position] = _elide_tool_message(newest[position]) + shrunk[-1] = newest + if sum(_unit_tokens(unit) for unit in shrunk) <= budget: + break + if sum(_unit_tokens(unit) for unit in shrunk) > budget: + shrunk[-1] = _elide_unit(newest) + if sum(_unit_tokens(unit) for unit in shrunk) > budget: + logger.warning( + "Current turn exceeds the context budget even after eliding tool output " + "(estimated %s tokens, budget %s).", + sum(_unit_tokens(unit) for unit in shrunk), + budget, + ) + return shrunk + + +# ------------------------------------------------------------- react loop + + +@dataclass +class _TurnState: + thoughts: list[str] = field(default_factory=list) + used_tools: bool = False + empty_retries: int = 0 + pending_nudge: bool = False + prompt_tokens: int = 0 + completion_tokens: int = 0 + total_tokens: int = 0 + usage_reported: bool = True + # Every model call of the turn (usage) vs. the calls that count toward + # max_iterations (decision D-02: summarisation does not count). + model_calls: int = 0 + budget_used: int = 0 + # Loop guard: identical (tool, arguments) calls since the last change. + call_counts: dict[str, int] = field(default_factory=dict) + forced_final: str | None = None + # Transient notes for the next request only. + notes: list[str] = field(default_factory=list) + after_compaction: bool = False + # Tool-call sequence numbers: when the workspace last changed and when a + # command last ran, for verify-on-stop. + step: int = 0 + last_change_step: int = 0 + last_command_step: int = 0 + commands_run: list[str] = field(default_factory=list) + verify_nudged: bool = False + continuation_nudges: int = 0 + step_at_last_continuation: int = -1 + pending_continuation_flag: bool = False + verify_runs: int = 0 + verification_note: str | None = None + reviewed: bool = False + compaction_failed: bool = False + # Changes made through IDE tools (MCP refactorings) have no file diff. + external_changes: bool = False + side_call_failures: list[dict[str, Any]] = field(default_factory=list) + # Reasoning fields of the latest model response, attached (once) to the + # first stored message that holds that response's output. + reasoning_fields: dict[str, Any] | None = None + # An IDE build/run (MCP) after the last change: enough to skip the verify + # nudge, never a replacement for the profile's verification command. + last_ide_run_step: int = 0 + + @property + def usage(self) -> dict[str, int] | None: + if not self.usage_reported or self.model_calls == 0: + return None + return { + "prompt_tokens": self.prompt_tokens, + "completion_tokens": self.completion_tokens, + "total_tokens": self.total_tokens, + } + + +_LOOP_NUDGE = ( + "You have called {name} with identical arguments {count} times in this turn and the result " + "has not changed. Do not repeat it: use the result you already have, change the arguments " + "or the approach, or stop and explain what is blocking you." +) +_LOOP_STOP_NOTICE = ( + "Tools are now disabled for this turn because the same call was repeated {count} times " + "without progress. Reply with a user-facing message: what you did, what did not work, and " + "what the user could do next." +) +_LOOP_NUDGE_AT = 3 +_LOOP_STOP_AT = 5 +_LOOP_NOT_RUN = "Not run: tools are disabled for this turn after repeated identical calls." +_LOOP_STOPPED_TEXT = ( + "I stopped because the same step kept repeating without progress, so tools were disabled " + "for the rest of this turn. Send another message with more guidance to continue." +) +_MAX_CONTINUATIONS = 2 +_CUT_OFF_NOTE = ( + "[Runtime check, not a message from the user] Your last message was cut off by the output " + "limit. Continue where it stopped: call the tools you need, or finish your final answer." +) +_CONTINUE_NOTE = ( + "[Runtime check, not a message from the user] Your last message announced more work but " + "called no tool, so the turn would end here. Continue: call the tools you need, or, if " + "you are done, give your final answer." +) +_VERIFY_NUDGE = ( + "[Runtime check, not a message from the user] You changed {files} but ran no command since " + "your last change. If a test, build or lint command applies, run it now and fix what fails; " + "otherwise give your final answer and name the command the user should run." +) +_VERIFY_FAILED_NOTE = ( + "[Runtime check, not a message from the user] The runtime ran the configured verification " + "command `{command}` after your changes and it failed ({outcome}). Fix the failures, or " + "explain in your final answer why they are unrelated to your change." +) +_REVIEW_NOTE = ( + "[Self-review by the Code4Me runtime, not a message from the user] A separate review of " + "your diff for this request raised these possible problems:\n{issues}\n" + "Fix the real ones (and re-run verification if you changed code), then give your final " + "answer. If a point is wrong, say so briefly instead of changing the code." +) +_REVIEW_SYSTEM_PROMPT = ( + "You review a change made by an AI coding agent. You see the user's request, the unified " + "diff of the agent's changes, the commands it ran, and its final message. Report only real " + "problems: bugs, syntax or import errors, broken callers, logic that does not do what the " + "request asks, parts of the request left undone, changes the request did not ask for, and " + "claims in the final message that the diff and commands do not support (for example tests " + "said to pass that were never run). Ignore style, naming and formatting. If there is nothing " + "important, reply with exactly NO_ISSUES. Otherwise list at most five issues, one per line, " + "each starting with '- ' and naming the file (and line when you can)." +) +_SUMMARY_SYSTEM_PROMPT = ( + "You compress the earlier part of a coding-agent session into a checkpoint. The original " + "messages are removed afterwards, so the agent will rely only on your checkpoint. Keep " + "facts, not prose, under these headings: Goal; User requirements and constraints; Work done " + "(files changed and how, commands run and their results); Current state; Decisions; Errors " + "and open problems (quote error messages verbatim); Next steps. Use short bullet points, " + "never invent anything, and stay under 350 words." +) +_SUMMARY_INPUT_CHARS = 60_000 +_REVIEW_DIFF_CHARS = 30_000 + + +class OpenAICompatibleReactAdapter: + def __init__( + self, + config: AgentConfig, + *, + telemetry: AgentTelemetryRecorder, + tool_registry: ToolRegistry, + event_sink: AgentEventSink | None = None, + session_state: SessionToolState | None = None, + ) -> None: + self._config = config + self._telemetry = telemetry + self._tool_registry = tool_registry + self._event_sink = event_sink or NoopAgentEventSink() + self._provider_instance: FakeOpenAICompatibleProvider | OpenAICompatibleProvider | None = None + self._session_state = ( + session_state + or getattr(tool_registry, "session_state", None) + or SessionToolState() + ) + + @property + def _harness(self) -> HarnessOptions: + return getattr(self._config, "harness", None) or HarnessOptions() + + # ------------------------------------------------------------ prompt + + def handle_prompt( + self, + *, + prompt: str, + run_id: str, + request_id: str, + message_id: str | None, + memory: "MemoryWindow | None" = None, + cancellation_event: Event | None = None, + ) -> AdapterResult: + if memory is None: + memory = MemoryWindow( + strategy=self._config.adapter.memory_window.strategy, + max_messages=self._config.adapter.memory_window.max_messages, + max_tokens=self._config.adapter.memory_window.max_tokens, + ) + command = slash_commands.parse(prompt, managed=bool(self._config.managed_mode)) + if command is not None: + return self._run_slash_command( + command, + memory=memory, + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + ) + provider = self._provider() + tool_names = sorted(self._tool_registry.known_tool_names()) + instructions = self._project_instructions() + profile = self._prompt_profile() + memory.ensure_system_message( + self._system_context( + tool_names=tool_names, instructions=instructions, prompt_profile=profile + ) + ) + prior_messages = max(0, memory.message_count() - 1) + memory.append({"role": "user", "content": prompt}) + self._session_state.begin_turn(run_id, prompt) + try: + return self._run_turn( + provider, + prompt=prompt, + run_id=run_id, + request_id=request_id, + memory=memory, + cancellation_event=cancellation_event, + prior_messages=prior_messages, + profile=profile, + instructions=instructions, + ) + finally: + self._session_state.end_turn() + + def _run_turn( + self, + provider: FakeOpenAICompatibleProvider | OpenAICompatibleProvider, + *, + prompt: str, + run_id: str, + request_id: str, + memory: MemoryWindow, + cancellation_event: Event | None, + prior_messages: int, + profile: str, + instructions: prompting.ProjectInstructions | None, + ) -> AdapterResult: + definitions = self._tool_registry.definitions() + reserve_tokens = _estimate_text_tokens(json.dumps(definitions)) + _OUTPUT_HEADROOM_TOKENS + state = _TurnState() + max_iterations = max(1, int(self._config.adapter.max_iterations)) + harness = self._harness + + while True: + if _turn_was_cancelled(cancellation_event): + return self._cancelled(memory, state) + if harness.context_summarization and not state.compaction_failed: + compacted = self._maybe_compact( + provider, + memory, + state, + reserve_tokens=reserve_tokens, + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + ) + if compacted == "cancelled": + return self._cancelled(memory, state) + if compacted == "failed": + state.compaction_failed = True # do not retry every iteration + trimmed = memory.compact_current_turn(reserve_tokens=reserve_tokens) + # Reported as integer metrics on this iteration's agent.model.requested + # (no event type of its own: the study counts steps as events). + request_metrics = ( + { + "context_elided_units": trimmed["elided_units"], + "context_tokens_before": trimmed["estimated_tokens_before"], + "context_tokens_after": trimmed["estimated_tokens_after"], + "context_budget_tokens": trimmed["budget_tokens"], + "token_scale_milli": int(round(memory.scale * 1000)), + } + if trimmed + else None + ) + iteration = state.budget_used + 1 + final_round = iteration >= max_iterations or state.forced_final is not None + request_messages = memory.window(reserve_tokens=reserve_tokens) + notes: list[str] = [] + tool_choice: str | None = "auto" + if state.forced_final is not None: + notes.append(state.forced_final) + tool_choice = "none" + elif final_round and max_iterations > 1: + notes.append(_BUDGET_NOTICE) + tool_choice = "none" + else: + if state.pending_nudge: + notes.append(_EMPTY_RESPONSE_NUDGE) + state.pending_nudge = False + notes.extend(state.notes) + state.notes = [] + reminded = bool( + harness.instruction_reminders + and prompting.should_remind( + iteration, prior_messages, after_compaction=state.after_compaction + ) + ) + if reminded: + notes.append(self._reminder(max_iterations - iteration + 1, instructions)) + state.after_compaction = False + if not definitions: + tool_choice = None + transient = [{"role": "user", "content": "\n\n".join(notes)}] if notes else [] + messages = request_messages + transient + + payload: dict[str, Any] = { + "iteration": iteration, + "message_count": len(messages), + "approx_token_count": sum(_estimate_tokens(message) for message in messages), + "tool_choice": tool_choice, + "final_round": final_round, + "call_purpose": "turn", + } + if iteration == 1: + payload["prompt_profile"] = profile + if instructions is not None: + payload.update(instructions.telemetry()) + if reminded: + payload["reminder"] = True + if state.forced_final is not None: + payload["loop_guard"] = "forced_stop" + if state.pending_continuation_flag: + payload["continuation_nudge"] = True + state.pending_continuation_flag = False + self._telemetry.record( + event_type="agent.model.requested", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload=payload, + metrics=request_metrics, + raw_payload={"messages": messages}, + ) + started_at = perf_counter() + try: + turn = provider.generate( + messages, + run_id=run_id, + tool_choice=tool_choice, + cancellation_event=cancellation_event, + ) + except ProviderCancelled: + return self._cancelled(memory, state) + except FakeProviderExhaustedError: + return self._fail( + memory, + state, + run_id=run_id, + request_id=request_id, + failure_reason="provider_exhausted", + stop_reason="provider_exhausted", + text="The fake provider stopped before returning a final answer.", + ) + except ProviderRequestFailed as exc: + return self._fail( + memory, + state, + run_id=run_id, + request_id=request_id, + failure_reason="provider_request_failed", + stop_reason="error", + text=_provider_error_text(exc), + failure=_model_failure_details(exc, started_at=started_at, purpose="turn"), + ) + except Exception as exc: # noqa: BLE001 + return self._fail( + memory, + state, + run_id=run_id, + request_id=request_id, + failure_reason="provider_request_failed", + stop_reason="error", + text=f"The model request failed: {exc}", + failure=_model_failure_details(exc, started_at=started_at, purpose="turn"), + ) + state.budget_used += 1 + self._account_model_call( + state, + turn, + run_id=run_id, + request_id=request_id, + duration_ms=(perf_counter() - started_at) * 1000, + iteration=iteration, + context_budget=memory.max_tokens, + ) + if not turn.usage_estimated: + memory.calibrate( + provider_prompt_tokens=int(turn.usage.get("prompt_tokens", 0)), + estimated_tokens=payload["approx_token_count"] + reserve_tokens - _OUTPUT_HEADROOM_TOKENS, + ) + if _turn_was_cancelled(cancellation_event): + return self._cancelled(memory, state) + + parsed = self._parse_output(turn, run_id=run_id, request_id=request_id) + if parsed is None: + return self._fail( + memory, + state, + run_id=run_id, + request_id=request_id, + failure_reason="parse_failed", + stop_reason="error", + text=_PARSE_ERROR_TEXT, + ) + state.reasoning_fields = parsed.reasoning_fields + if parsed.thought: + state.thoughts.append(parsed.thought) + emit_event( + self._event_sink, + "thought", + ThoughtEvent( + run_id=run_id, + request_id=request_id, + text=parsed.thought, + phase="completed", + duration_ms=round((perf_counter() - started_at) * 1000, 3), + ), + ) + + if parsed.tool_calls and state.forced_final is not None: + # Tools were disabled after repeated identical calls; a provider + # that ignores tool_choice="none" must not run them again. The + # never-run calls are runtime-only: a reopened chat skips them + # (the model's text is part of the final message below). + memory.append( + _with_reasoning( + _assistant_tool_call_message(parsed.tool_calls, text=None, runtime="loop_stop"), state + ) + ) + for call in parsed.tool_calls: + memory.append(_tool_message(call, _not_run_result(call, _LOOP_NOT_RUN))) + text = f"{parsed.text}\n\n{_LOOP_STOPPED_TEXT}" if parsed.text else _LOOP_STOPPED_TEXT + return self._finish( + memory, + state, + run_id=run_id, + request_id=request_id, + text=text, + stop_reason="loop_detected", + iteration=iteration, + emit=True, + ) + + if parsed.tool_calls: + if parsed.text: + self._emit_text(parsed.text, run_id=run_id, request_id=request_id, final=False, iteration=iteration) + memory.append(_with_reasoning(_assistant_tool_call_message(parsed.tool_calls, text=parsed.text), state)) + state.used_tools = True + results = self._execute_batch( + parsed.tool_calls, + memory=memory, + state=state, + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + max_chars=_batch_result_cap(memory.max_tokens, len(parsed.tool_calls)), + ) + if results is None: + return self._cancelled(memory, state) + question = _asked_question(parsed.tool_calls, results) + if question is not None: + return self._finish( + memory, + state, + run_id=run_id, + request_id=request_id, + text=question, + stop_reason="awaiting_user", + iteration=iteration, + emit=True, + ) + self._update_loop_guard(state, parsed.tool_calls, results) + if final_round: + return self._finish( + memory, + state, + run_id=run_id, + request_id=request_id, + text=_budget_exhausted_text(parsed.tool_calls, results), + stop_reason="loop_detected" if state.forced_final else "max_turn_requests", + iteration=iteration, + emit=True, + ) + continue + + cut_off = parsed.finish_reason == "length" + if ( + getattr(self._config, "autonomous", False) + and parsed.text + and state.used_tools + and state.forced_final is None + and not final_round + and state.continuation_nudges < _MAX_CONTINUATIONS + and state.step > state.step_at_last_continuation + and max_iterations - state.budget_used >= 2 + and (cut_off or _announces_more_work(parsed.text)) + ): + # Headless only: "Let me also check…" (or an answer cut off at + # the output limit) with no tool call would end the turn + # mid-task. At most two nudges, each needing a tool run since + # the last one, so the model can always end the turn. + state.continuation_nudges += 1 + state.step_at_last_continuation = state.step + state.pending_continuation_flag = True + self._continue_with_note( + memory, + parsed.text, + note=_CUT_OFF_NOTE if cut_off else _CONTINUE_NOTE, + kind="continue", + run_id=run_id, + request_id=request_id, + iteration=iteration, + state=state, + ) + continue + if parsed.text: + if state.forced_final is not None: + stop_reason = "loop_detected" + elif final_round and max_iterations > 1 and state.used_tools: + stop_reason = "max_turn_requests" + else: + stop_reason = _stop_reason_from_finish(parsed.finish_reason) + gate = self._stop_gate( + provider, + memory, + state, + prompt=prompt, + answer=parsed.text, + max_iterations=max_iterations, + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + ) + if gate == "cancelled": + return self._cancelled(memory, state) + if gate == "continue": + continue + text = parsed.text + if state.verification_note: + text = f"{text}\n\n{state.verification_note}" + return self._finish( + memory, + state, + run_id=run_id, + request_id=request_id, + text=text, + stop_reason=stop_reason, + iteration=iteration, + emit=True, + ) + + if state.empty_retries == 0 and not final_round: + state.empty_retries = 1 + state.pending_nudge = True + continue + return self._fail( + memory, + state, + run_id=run_id, + request_id=request_id, + failure_reason="empty_response", + stop_reason="error", + text=_EMPTY_RESPONSE_TEXT, + ) + + # -------------------------------------------------------- model calls + + def _account_model_call( + self, + state: _TurnState, + turn: ProviderTurn, + *, + run_id: str, + request_id: str, + duration_ms: float, + iteration: int, + context_budget: int, + purpose: str = "turn", + ) -> None: + state.model_calls += 1 + self._record_model_completed( + run_id=run_id, + request_id=request_id, + duration_ms=duration_ms, + provider_turn=turn, + purpose=purpose, + ) + if turn.usage_estimated: + state.usage_reported = False + return + state.prompt_tokens += turn.usage.get("prompt_tokens", 0) + state.completion_tokens += turn.usage.get("completion_tokens", 0) + state.total_tokens += turn.usage.get("total_tokens", 0) + if purpose != "turn": + # A summary or review call runs on a different, small context: an + # IDE context meter must keep showing the conversation's usage. + return + emit_event( + self._event_sink, + "usage", + UsageEvent( + run_id=run_id, + request_id=request_id, + iteration=iteration, + model=turn.model, + prompt_tokens=turn.usage.get("prompt_tokens", 0), + completion_tokens=turn.usage.get("completion_tokens", 0), + total_tokens=turn.usage.get("total_tokens", 0), + turn_total_tokens=state.total_tokens, + context_budget_tokens=context_budget, + ), + ) + + def _side_call( + self, + provider: FakeOpenAICompatibleProvider | OpenAICompatibleProvider, + messages: list[dict[str, Any]], + state: _TurnState, + *, + purpose: str, + run_id: str, + request_id: str, + cancellation_event: Event | None, + context_budget: int, + extra_payload: dict[str, Any] | None = None, + ) -> str | None: + """A tool-less model call outside the conversation (summary, review). + + Returns the text, ``None`` when the call failed (the turn carries on + without it) and raises ``ProviderCancelled`` on cancellation. + """ + self._telemetry.record( + event_type="agent.model.requested", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "iteration": state.budget_used, + "message_count": len(messages), + "approx_token_count": sum(_estimate_tokens(message) for message in messages), + "tool_choice": None, + "final_round": False, + "call_purpose": purpose, + **(extra_payload or {}), + }, + raw_payload={"messages": messages}, + ) + started_at = perf_counter() + try: + turn = provider.generate( + messages, + run_id=run_id, + tool_choice=None, + cancellation_event=cancellation_event, + include_tools=False, + ) + except ProviderCancelled: + raise + except Exception as exc: # noqa: BLE001 - the main loop continues without it + logger.warning("The %s model call failed: %s", purpose, exc) + payload, metrics = _model_failure_details(exc, started_at=started_at, purpose=purpose) + state.side_call_failures.append( + { + "call_purpose": purpose, + "error_type": payload["error_type"], + **{key: metrics[key] for key in ("status_code", "attempts") if key in metrics}, + } + ) + return None + self._account_model_call( + state, + turn, + run_id=run_id, + request_id=request_id, + duration_ms=(perf_counter() - started_at) * 1000, + iteration=max(1, state.budget_used), + context_budget=context_budget, + purpose=purpose, + ) + output = turn.output.get("output", turn.output) if isinstance(turn.output, dict) else {} + text = output.get("final_answer", output.get("text")) if isinstance(output, dict) else None + return text.strip() if isinstance(text, str) and text.strip() else None + + # ----------------------------------------------------- summarisation + + def _maybe_compact( + self, + provider: FakeOpenAICompatibleProvider | OpenAICompatibleProvider, + memory: MemoryWindow, + state: _TurnState, + *, + reserve_tokens: int, + run_id: str, + request_id: str, + cancellation_event: Event | None, + keep_recent_units: int = 4, + force: bool = False, + ) -> str | None: + """Summarise the oldest units when elision alone no longer fits (D1).""" + count = memory.compaction_plan( + reserve_tokens=reserve_tokens, keep_recent_units=keep_recent_units + ) + if count <= 0 and force: + count = max(0, memory.older_unit_count() - keep_recent_units) + if count <= 0: + return None + units = memory.oldest_units(count) + rendered = _render_units_for_summary(units, max_chars=min(_SUMMARY_INPUT_CHARS, memory.max_tokens * 2)) + messages = [ + {"role": "system", "content": _SUMMARY_SYSTEM_PROMPT}, + {"role": "user", "content": f"Session excerpt to compress:\n\n{rendered}"}, + ] + try: + summary = self._side_call( + provider, + messages, + state, + purpose="summarize", + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + context_budget=memory.max_tokens, + extra_payload={"compacted_units": count}, + ) + except ProviderCancelled: + return "cancelled" + if not summary: + return "failed" + memory.compact(count, summary) + state.after_compaction = True + return "compacted" + + # --------------------------------------------------------- stop gates + + def _stop_gate( + self, + provider: FakeOpenAICompatibleProvider | OpenAICompatibleProvider, + memory: MemoryWindow, + state: _TurnState, + *, + prompt: str, + answer: str, + max_iterations: int, + run_id: str, + request_id: str, + cancellation_event: Event | None, + ) -> str: + """Before a final answer: verify (C5), then self-review (E3). + + Returns ``finish``, ``continue`` (the loop goes on with a note in + memory) or ``cancelled``. A continuation needs room to act: two calls + after a verification problem (fix, answer), three for a review + (review, fix, answer); the last call of a turn has no tools. + """ + changed = self._session_state.changed_paths() + if not changed and not state.external_changes: + return "finish" + remaining = max_iterations - state.budget_used + harness = self._harness + if harness.verify_on_stop and remaining >= 2 and state.last_change_step > state.last_command_step: + command = self._verify_command() + if command is not None and state.verify_runs < 2: + state.verify_runs += 1 + outcome = self._run_verification( + command, + memory, + state, + answer=answer, + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + ) + if outcome in {"cancelled", "continue"}: + return outcome + elif ( + command is None + and not state.verify_nudged + and self._can_run_commands() + and state.last_change_step > state.last_ide_run_step + ): + state.verify_nudged = True + self._continue_with_note( + memory, + answer, + note=_VERIFY_NUDGE.format( + files=(", ".join(changed[:5]) + (" and more" if len(changed) > 5 else "")) + or "files through IDE tools", + ), + kind="verify", + run_id=run_id, + request_id=request_id, + iteration=state.budget_used, + state=state, + ) + return "continue" + if harness.self_review and not state.reviewed and remaining >= 3: + state.reviewed = True + diff = self._session_state.turn_diff(max_chars=_REVIEW_DIFF_CHARS) + if not diff.strip(): + return "finish" + calls_before = state.model_calls + try: + issues = self._self_review( + provider, + state, + prompt=prompt, + answer=answer, + diff=diff, + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + context_budget=memory.max_tokens, + ) + except ProviderCancelled: + return "cancelled" + if state.model_calls > calls_before: + state.budget_used += 1 # decision D-02: the review counts + if issues: + self._emit_text( + f"Self-review of the changes raised possible issues:\n{issues}", + run_id=run_id, + request_id=request_id, + final=False, + iteration=state.budget_used, + ) + self._continue_with_note( + memory, + answer, + note=_REVIEW_NOTE.format(issues=issues), + kind="review", + run_id=run_id, + request_id=request_id, + iteration=state.budget_used, + emit_answer=False, + state=state, + ) + return "continue" + return "finish" + + def _continue_with_note( + self, + memory: MemoryWindow, + answer: str, + *, + note: str, + kind: str, + run_id: str, + request_id: str, + iteration: int, + emit_answer: bool = True, + state: _TurnState | None = None, + ) -> None: + """Keep the model's provisional answer and add a runtime note; the loop continues.""" + provisional: dict[str, Any] = {"role": "assistant", "content": answer} + if state is not None: + provisional = _with_reasoning(provisional, state) + if emit_answer: + self._emit_text(answer, run_id=run_id, request_id=request_id, final=False, iteration=iteration) + else: + # The model sees its provisional answer; the user never did, so a + # reopened chat must not replay it. + provisional["code4me_runtime"] = "provisional" + memory.append(provisional) + memory.append({"role": "user", "content": note, "code4me_runtime": kind}) + + def _verify_command(self) -> list[str] | None: + """The profile's verification argv, when this session may actually run it. + + A per-user config row can block more programs, and the program may not + be installed here; either falls back to asking the model to verify. + """ + command = self._harness.verify_command + if not command or not self._can_run_commands(): + return None + from code4me2_agent.command_tools import available_commands, blocked_program + + if blocked_program(command, self._config.commands.blocked_commands) is not None: + return None + if not available_commands([Path(command[0]).name]): + return None + return list(command) + + def _can_run_commands(self) -> bool: + if self._config.approval_policy == "suggestion_only": + return False + return "run_command" in self._tool_registry.known_tool_names() + + def _commands_status(self) -> str: + """The ``/status`` description of what run_command may start.""" + if not self._can_run_commands(): + return "none" + blocked = self._config.commands.blocked_commands + return "any installed program" + (f" except {', '.join(blocked)}" if blocked else "") + + def _run_verification( + self, + command: list[str], + memory: MemoryWindow, + state: _TurnState, + *, + answer: str, + run_id: str, + request_id: str, + cancellation_event: Event | None, + ) -> str: + """Run the profile's verification command; ``continue`` when it failed.""" + call = ToolCall( + tool_call_id=f"verify-{run_id[:8]}-{state.verify_runs}", + name="run_command", + arguments={"argv": list(command)}, + ) + result = self._run_tool_call( + call, + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + max_chars=_batch_result_cap(memory.max_tokens, 1), + ) + if _turn_was_cancelled(cancellation_event): + memory.append(_assistant_tool_call_message([call], text=None, runtime="verify")) + memory.append(_tool_message(call, result)) + return "cancelled" + status = result.get("status") + exit_code = result.get("exit_code") + shown = " ".join(command) + state.step += 1 + state.last_command_step = state.step + state.commands_run.append(shown) + failed = (status == "ok" and exit_code not in (0, None)) or status == "timeout" + if not failed: + memory.append(_assistant_tool_call_message([call], text=None, runtime="verify")) + memory.append(_tool_message(call, result)) + if status == "ok" and exit_code == 0: + state.verification_note = f"Verified by the runtime: `{shown}` passed." + else: + # Not run (denied, rejected, not installed): not a failure of the change. + state.verification_note = ( + f"The runtime could not run the verification command `{shown}` " + f"({result.get('reason') or result.get('error_code') or status})." + ) + return "passed" if status == "ok" else "skipped" + outcome = f"exit code {exit_code}" if status == "ok" else "timed out" + self._emit_text(answer, run_id=run_id, request_id=request_id, final=False, iteration=state.budget_used) + memory.append(_with_reasoning({"role": "assistant", "content": answer}, state)) + memory.append(_assistant_tool_call_message([call], text=None, runtime="verify")) + memory.append(_tool_message(call, result)) + memory.append( + { + "role": "user", + "content": _VERIFY_FAILED_NOTE.format(command=shown, outcome=outcome), + "code4me_runtime": "verify", + } + ) + state.verification_note = None + return "continue" + + def _self_review( + self, + provider: FakeOpenAICompatibleProvider | OpenAICompatibleProvider, + state: _TurnState, + *, + prompt: str, + answer: str, + diff: str, + run_id: str, + request_id: str, + cancellation_event: Event | None, + context_budget: int, + ) -> str | None: + """One clean-context review call; returns the issues, or None when there are none.""" + commands = "\n".join(f"- {command}" for command in state.commands_run[-10:]) or "(none)" + messages = [ + {"role": "system", "content": _REVIEW_SYSTEM_PROMPT}, + { + "role": "user", + "content": ( + f"User request:\n{prompt[:6000]}\n\nDiff of the agent's changes:\n```diff\n{diff}```\n\n" + f"Commands the agent ran this turn:\n{commands}\n\nAgent's final message:\n{answer[:4000]}" + ), + }, + ] + text = self._side_call( + provider, + messages, + state, + purpose="self_review", + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + context_budget=context_budget, + ) + return _review_issues(text) + + # ------------------------------------------------------- tool batches + + def _execute_batch( + self, + tool_calls: list[ToolCall], + *, + memory: MemoryWindow, + state: _TurnState, + run_id: str, + request_id: str, + cancellation_event: Event | None, + max_chars: int, + ) -> list[dict[str, Any]] | None: + """Run a batch in order; runs of read-only calls execute concurrently (E2). + + Results are appended to memory in call order. Returns None when the + turn was cancelled (every call still gets a result message). + """ + results: list[dict[str, Any]] = [] + index = 0 + asked = False + while index < len(tool_calls): + if _turn_was_cancelled(cancellation_event): + for remaining in tool_calls[index:]: + memory.append(_tool_message(remaining, _cancelled_tool_result(remaining))) + return None + if asked: + skipped = { + "tool_name": tool_calls[index].name, + "tool_call_id": tool_calls[index].tool_call_id, + "status": "skipped", + "reason": "Not run: the turn ended with a question to the user.", + } + results.append(skipped) + memory.append(_tool_message(tool_calls[index], skipped)) + index += 1 + continue + segment = [index] + if self._harness.parallel_tools and self._parallel_safe(tool_calls[index]): + while ( + segment[-1] + 1 < len(tool_calls) + and len(segment) < 8 + and self._parallel_safe(tool_calls[segment[-1] + 1]) + ): + segment.append(segment[-1] + 1) + if len(segment) > 1: + + def run_unless_cancelled(call: ToolCall) -> dict[str, Any]: + if _turn_was_cancelled(cancellation_event): + return _cancelled_tool_result(call) + return self._run_tool_call( + call, + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + max_chars=max_chars, + ) + + with ThreadPoolExecutor(max_workers=min(4, len(segment)), thread_name_prefix="code4me2-tool") as pool: + futures = [pool.submit(run_unless_cancelled, tool_calls[position]) for position in segment] + segment_results = [future.result() for future in futures] + else: + segment_results = [ + self._run_tool_call( + tool_calls[index], + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + max_chars=max_chars, + ) + ] + for position, result in zip(segment, segment_results): + call = tool_calls[position] + results.append(result) + memory.append(_tool_message(call, result)) + self._note_step(state, call, result) + if call.name == "ask_user" and result.get("status") == "ok": + asked = True + index = segment[-1] + 1 + return results + + def _parallel_safe(self, tool_call: ToolCall) -> bool: + if tool_call.argument_error: + return False + check = getattr(self._tool_registry, "is_parallel_safe", None) + return bool(check(tool_call.name)) if callable(check) else False + + def _tool_class(self, name: str) -> str | None: + """"edit" / "execute" for built-in or IDE (MCP) tools that change or run things.""" + if name in _WORKSPACE_MUTATIONS: + return "edit" + if name == "run_command": + return "execute" + access = getattr(self._tool_registry, "mcp_access", None) + value = access(name) if callable(access) and name.startswith("mcp__") else None + return value if value in ("edit", "execute") else None + + def _note_step(self, state: _TurnState, tool_call: ToolCall, result: dict[str, Any]) -> None: + state.step += 1 + status = result.get("status") + tool_class = self._tool_class(tool_call.name) + if tool_class == "edit" and (status == "ok" or _wrote_revision(result)): + state.last_change_step = state.step + # A verification result describes the code as it was; any later + # change voids it (the gate verifies again when it can). + state.verification_note = None + if tool_call.name.startswith("mcp__"): + state.external_changes = True + elif tool_class == "execute" and tool_call.name.startswith("mcp__") and status == "ok": + state.last_ide_run_step = state.step + state.commands_run.append(f"IDE: {tool_call.name.split('__', 2)[-1]}") + elif tool_call.name == "run_command" and status in {"ok", "error", "timeout"}: + argv = tool_call.arguments.get("argv") + if isinstance(argv, str): + argv = [argv] + # Verify-on-stop: inspection (git status/diff, ls, cat, grep…) or file + # shuffling after the last change is not verification; a command that + # ran the code is, whatever its exit code (the model saw the output). + if status == "ok" and isinstance(argv, list) and _is_verification_command(argv): + state.last_command_step = state.step + if isinstance(argv, list): + state.commands_run.append(" ".join(str(item) for item in argv)) + + def _update_loop_guard( + self, state: _TurnState, tool_calls: list[ToolCall], results: list[dict[str, Any]] + ) -> None: + if not self._harness.loop_guard: + return + worst, worst_name = 0, "" + for call, result in zip(tool_calls, results): + if self._tool_class(call.name) == "edit" and ( + result.get("status") == "ok" or _wrote_revision(result) + ): + # The workspace changed: repeating a read or a test is progress. + state.call_counts.clear() + worst, worst_name = 0, "" + continue + key = call.name + "\0" + json.dumps(call.arguments, sort_keys=True, default=str) + count = state.call_counts.get(key, 0) + 1 + state.call_counts[key] = count + if count > worst: + worst, worst_name = count, call.name + if worst >= _LOOP_STOP_AT: + state.forced_final = _LOOP_STOP_NOTICE.format(count=worst) + elif worst >= _LOOP_NUDGE_AT: + state.notes.append(_LOOP_NUDGE.format(name=worst_name, count=worst)) + + # --------------------------------------------------------- outcomes + + def _emit_text( + self, text: str, *, run_id: str, request_id: str, final: bool, iteration: int + ) -> None: + emit_event( + self._event_sink, + "assistant_text", + AssistantTextEvent( + run_id=run_id, + request_id=request_id, + message_id=request_id, + text=text, + final=final, + iteration=iteration, + ), + ) + + def _finish( + self, + memory: MemoryWindow, + state: _TurnState, + *, + run_id: str, + request_id: str, + text: str, + stop_reason: str, + iteration: int, + emit: bool, + ) -> AdapterResult: + if emit: + self._emit_text(text, run_id=run_id, request_id=request_id, final=True, iteration=iteration) + memory.append(_with_reasoning({"role": "assistant", "content": text}, state)) + return AdapterResult( + final_response=text, + stop_reason=stop_reason, + run_status="completed", + thoughts=tuple(state.thoughts), + response_emitted=True, + usage=state.usage, + side_call_failures=tuple(state.side_call_failures), + ) + + def _fail( + self, + memory: MemoryWindow, + state: _TurnState, + *, + run_id: str, + request_id: str, + failure_reason: str, + stop_reason: str, + text: str, + failure: tuple[dict[str, Any], dict[str, Any]] | None = None, + ) -> AdapterResult: + extra_payload, metrics = failure if failure is not None else (None, None) + self._record_loop_failure( + run_id=run_id, + request_id=request_id, + failure_reason=failure_reason, + extra_payload=extra_payload, + metrics=metrics, + ) + self._emit_text(text, run_id=run_id, request_id=request_id, final=True, iteration=state.model_calls) + memory.append({"role": "assistant", "content": text}) + return AdapterResult( + final_response=text, + stop_reason=stop_reason, + run_status="failed", + thoughts=tuple(state.thoughts), + response_emitted=True, + usage=state.usage, + side_call_failures=tuple(state.side_call_failures), + ) + + def _cancelled(self, memory: MemoryWindow, state: _TurnState) -> AdapterResult: + snapshot = memory.snapshot() + if not snapshot or snapshot[-1].get("role") != "assistant" or snapshot[-1].get("tool_calls"): + memory.append( + { + "role": "assistant", + "content": "[Cancelled by the user before the turn completed.]", + } + ) + result = _cancelled_result(state.thoughts) + return AdapterResult( + final_response=result.final_response, + stop_reason=result.stop_reason, + run_status=result.run_status, + thoughts=result.thoughts, + response_emitted=True, + usage=state.usage, + side_call_failures=tuple(state.side_call_failures), + ) + + # -------------------------------------------------------- tool calls + + def _run_tool_call( + self, + tool_call: ToolCall, + *, + run_id: str, + request_id: str, + cancellation_event: Event | None, + max_chars: int = _MAX_TOOL_RESULT_CHARS, + ) -> dict[str, Any]: + self._telemetry.record( + event_type="agent.tool.called", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + "arguments": tool_call.arguments, + }, + raw_payload={ + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + "arguments": tool_call.arguments, + }, + ) + base = {"tool_name": tool_call.name, "tool_call_id": tool_call.tool_call_id} + if tool_call.argument_error: + self._tool_registry.report_invalid_arguments( + tool_call, run_id=run_id, request_id=request_id + ) + self._record_tool_failure( + run_id=run_id, + request_id=request_id, + tool_call=tool_call, + failure_reason="invalid_arguments", + error_message=tool_call.argument_error, + ) + result: dict[str, Any] = { + **base, + "status": "error", + "error_code": "invalid_arguments", + "error": tool_call.argument_error, + "hint": "Re-issue the call with a JSON object containing the required arguments.", + } + if tool_call.raw_arguments: + result["raw_arguments"] = tool_call.raw_arguments[:500] + return result + try: + output = self._tool_registry.execute( + tool_call, + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + ) + except ToolRegistryError as exc: + self._record_tool_denial( + run_id=run_id, + request_id=request_id, + tool_call=tool_call, + denial_reason=exc.failure_reason, + error_message=str(exc), + ) + if isinstance(exc, ToolRevisionRequested): + return {**base, **exc.result} + if exc.failure_reason == "approval_rejected": + return { + **base, + "status": "rejected", + "reason": "user_rejected", + "message": ( + "The user rejected this action. Do not retry this " + "requested change with another mutating tool; explain " + "that no change was made or ask what they prefer instead." + ), + } + return { + **base, + "status": "denied", + "reason": exc.failure_reason, + "error": str(exc), + "hint": self._denial_hint(exc.failure_reason), + } + except ToolArgumentError as exc: + self._record_tool_failure( + run_id=run_id, + request_id=request_id, + tool_call=tool_call, + failure_reason="invalid_arguments", + error_message=str(exc), + ) + result = { + **base, + "status": "error", + "error_code": "invalid_arguments", + "error": str(exc), + "hint": "Fix the named argument and call the tool again.", + } + if exc.field: + result["field"] = exc.field + return result + except PermissionError as exc: + if getattr(exc, "errno", None) is not None and not isinstance(exc, ToolError): + # An operating-system EACCES/EPERM, not a policy decision. + self._record_tool_failure( + run_id=run_id, + request_id=request_id, + tool_call=tool_call, + failure_reason="permission_denied", + error_message=str(exc), + ) + return { + **base, + "status": "error", + "error_code": "permission_denied", + "error": str(exc), + "hint": "The operating system refused access; check file permissions or choose another path.", + } + # Workspace-policy denials are recorded by the file/command tools + # themselves; a second event here would double-count the study's + # step tally. + return { + **base, + "status": "denied", + "reason": "workspace_policy", + "error": str(exc), + "hint": _hint_for(tool_call.name, exc), + } + except TimeoutError as exc: + self._record_tool_failure( + run_id=run_id, + request_id=request_id, + tool_call=tool_call, + failure_reason="timeout", + error_message=str(exc), + ) + return { + **base, + "status": "timeout", + "error": str(exc), + "hint": "The tool did not finish in time; retry with a narrower request or a longer timeout.", + } + except Exception as exc: # noqa: BLE001 + error_code = getattr(exc, "code", None) if isinstance(exc, ToolError) else None + self._record_tool_failure( + run_id=run_id, + request_id=request_id, + tool_call=tool_call, + failure_reason=str(error_code or "tool_execution_error"), + error_message=str(exc), + ) + return { + **base, + "status": "error", + "error_code": str(error_code or type(exc).__name__), + "error": str(exc), + "hint": _hint_for(tool_call.name, exc), + } + return _cap_tool_result(_finalize_tool_output(output, base), max_chars) + + def _denial_hint(self, failure_reason: str) -> str: + if failure_reason == "tool_not_allowed": + allowed = ", ".join(sorted(self._tool_registry.known_tool_names())) or "none" + return f"This tool is not enabled for this session; use one of: {allowed}." + if failure_reason == "approval_policy_denied": + return ( + "This session is suggestion-only: describe the change as a unified diff in your " + "reply instead of applying it." + ) + if failure_reason.startswith("approval_"): + return "The action was not approved; explain what you wanted to do and ask how to proceed." + if failure_reason == "unsupported_tool": + return "Use one of the tools listed in the system prompt." + return "Adjust the request or explain the limitation to the user." + + # ----------------------------------------------------------- helpers + + def _provider(self) -> FakeOpenAICompatibleProvider | OpenAICompatibleProvider: + if self._provider_instance is None: + self._provider_instance = self._build_provider() + return self._provider_instance + + def _build_provider(self) -> FakeOpenAICompatibleProvider | OpenAICompatibleProvider: + if self._config.adapter.fake_provider.enabled: + return FakeOpenAICompatibleProvider(self._config.adapter.fake_provider.script) + provider_config = self._config.adapter.provider + return OpenAICompatibleProvider( + kind=provider_config.kind, + base_url=provider_config.base_url, + model=provider_config.model, + api_key_env=provider_config.api_key_env, + timeout_seconds=provider_config.timeout_seconds, + auth_headers=provider_config.auth_headers, + tool_definitions=self._tool_registry.definitions(), + temperature=provider_config.temperature, + session_id=self._config.session_id, + managed_request=self._config.managed_request, + max_output_tokens=getattr(provider_config, "max_output_tokens", None), + request_deadline_seconds=getattr(provider_config, "request_deadline_seconds", None), + extra_body=getattr(provider_config, "extra_body", None), + ) + + def _prompt_profile(self) -> str: + harness = getattr(self._config, "harness", None) + option = getattr(harness, "prompt_profile", "auto") if harness is not None else "auto" + return prompting.resolve_prompt_profile(option, self._config.adapter.provider.model) + + def _project_instructions(self) -> prompting.ProjectInstructions | None: + harness = getattr(self._config, "harness", None) + if harness is not None and not harness.project_instructions: + return None + try: + return prompting.load_project_instructions(self._config.workspace_root) + except Exception: # noqa: BLE001 - instructions are optional context + logger.debug("Could not read project instruction files.", exc_info=True) + return None + + def _reminder( + self, calls_left: int, instructions: prompting.ProjectInstructions | None + ) -> str: + names = self._tool_registry.known_tool_names() + return prompting.reminder_text( + approval_policy=self._config.approval_policy, + can_edit=bool(names & _WORKSPACE_MUTATIONS), + can_run="run_command" in names and self._can_run_commands(), + calls_left=max(1, calls_left), + has_instructions=instructions is not None, + ) + + def _system_context( + self, + *, + tool_names: Sequence[str] | None = None, + instructions: prompting.ProjectInstructions | None = None, + prompt_profile: str | None = None, + ) -> str: + config = self._config + if tool_names is None: + registry = getattr(self, "_tool_registry", None) + if registry is not None: + tool_names = sorted(registry.known_tool_names()) + elif config.allowed_tools is not None: + tool_names = sorted(config.allowed_tools) + elif getattr(config, "tools", None) is not None: + tool_names = sorted(config.tools or []) + else: + tool_names = tool_catalog.tool_names() + tool_names = list(tool_names) + names = set(tool_names) + if prompt_profile is None: + harness = getattr(config, "harness", None) + prompt_profile = prompting.resolve_prompt_profile( + getattr(harness, "prompt_profile", "auto"), config.adapter.provider.model + ) + workspace_root = config.workspace_root.as_posix() + os_name = platform.system() + today = date.today().isoformat() + blocked = list(config.commands.blocked_commands) + policy = config.approval_policy + budget = max(1, int(config.adapter.max_iterations)) + + if "run_command" in tool_names: + shell = _usable_shell(blocked) + shell_text = ( + f"no implicit shell (for pipes, redirects or cd run e.g. " + f'["{shell}", "-c", "..."])' + if shell + else "no shell (no pipes, redirects or cd)" + ) + commands_line = ( + "run_command runs a program installed on this machine, or a project script by path, " + f"with an argv list and {shell_text}." + ) + if blocked: + commands_line += ( + f" Blocked in this study: {', '.join(blocked)}. They are refused, also inside a " + "shell command; do not try to run them another way." + ) + else: + commands_line = "Commands cannot be run in this session." + if policy == "per_step": + policy_line = ( + "Approval policy: each file edit or command needs the user's approval in the IDE. " + "If the user rejects one, do not retry it another way; explain and ask how to proceed." + ) + elif policy == "suggestion_only": + policy_line = ( + "Approval policy: suggestion-only. You can read and search, but you cannot modify " + "files or run commands. Propose every change as a fenced unified diff with the " + "workspace-relative file path plus a one-line rationale, and list the commands the " + "user should run to verify." + ) + else: + policy_line = ( + "Approval policy: tools run immediately; file edits are applied as soon as you call them." + ) + plan_line = prompting.plan_guidance(prompt_profile) if "update_plan" in names else "" + tool_list = ", ".join(tool_names) if tool_names else "none" + # A researcher-authored profile prompt replaces the persona paragraph + # (ported from origin/sys_prompt). The operational instructions that + # follow (tools, approval policy, budget, how to work) are kept: the + # runtime cannot call tools correctly without them. + researcher_prompt = (config.adapter.system_prompt or "").strip() + if researcher_prompt: + persona = [ + researcher_prompt, + "", + "You are working inside the user's JetBrains IDE on the project at " + f"{workspace_root} (host OS: {os_name}; today: {today}). You act by calling tools; " + "the user sees your text and a card for every tool call.", + ] + else: + persona = [ + "You are Code4Me, a coding agent working inside the user's JetBrains IDE on the project at " + f"{workspace_root} (host OS: {os_name}; today: {today}). You act by calling tools; the user " + "sees your text and a card for every tool call.", + ] + lines = [ + *persona, + "", + f"Tools available in this session: {tool_list}.", + commands_line, + policy_line, + ] + if plan_line: + lines.append(plan_line) + lines += [ + f"Budget: at most {budget} model call{'s' if budget != 1 else ''} per user message. Each call " + "may issue several tool calls, so batch independent reads and searches together. On the " + "last call, tools are disabled and you must summarize.", + "", + "How to work", + "1. Explore first. Locate code with glob_files, grep_files or list_files, then read_file the " + "relevant parts. Never guess paths or file contents; if something does not exist, say so.", + prompting.edit_guidance(prompt_profile, names), + "3. Verify. When run_command is available, run the relevant tests, build or linter after " + "editing and fix what you broke. If you cannot verify, say what the user should run.", + "4. Keep going until the task is done or you are truly blocked. Do not ask for confirmation " + "of routine steps; ask only when an ambiguity would change the outcome" + + (", and then use ask_user." if "ask_user" in names else "."), + '5. Every tool result is JSON with a "status". On "error" or "denied", read the message and ' + 'hint, adjust, and try a different approach; never repeat an identical failing call. On ' + '"rejected", drop that change and ask what the user prefers.', + "6. Paths are workspace-relative (for example src/app.py). Do not invent absolute or " + "container paths.", + ] + notes = prompting.profile_notes(prompt_profile) + if notes: + lines += [f"{7 + index}. {note}" for index, note in enumerate(notes)] + lines += [ + "", + "Answering", + "- For greetings or general questions, answer directly without tools.", + "- A short sentence before a batch of tool calls is shown to the user as progress.", + "- Your final message states what changed (files), what was verified (commands and results) " + "and what remains or needs the user's decision. Do not paste code that tools already applied.", + ] + if instructions is not None: + lines += ["", prompting.instructions_block(instructions)] + return "\n".join(lines) + + # --------------------------------------------------------- slash commands + + def _run_slash_command( + self, + command: slash_commands.ParsedCommand, + *, + memory: MemoryWindow, + run_id: str, + request_id: str, + cancellation_event: Event | None, + ) -> AdapterResult: + state = _TurnState() + harness = self._harness + # Study sessions only parse /status and /undo (slash_commands.available), + # identically in every arm. Outside studies a command whose behaviour + # the configuration switched off answers neutrally. + if command.name == "status": + text = self._status_text(memory) + elif command.name == "undo": + text = self._undo_last_turn(memory) + elif command.name == "compact" and not harness.context_summarization: + text = "Compacting the conversation is not available in this session." + elif command.name == "review" and not harness.self_review: + text = "Reviews are not available in this session." + elif command.name == "compact": + text = self._compact_now(memory, state, run_id=run_id, request_id=request_id, cancellation_event=cancellation_event) + elif command.name == "review": + text = self._review_session( + memory, + state, + focus=command.argument, + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + ) + else: # pragma: no cover - parse() only returns known names + text = f"Unknown command /{command.name}." + if text is None: + return self._cancelled(memory, state) + self._emit_text(text, run_id=run_id, request_id=request_id, final=True, iteration=max(1, state.model_calls)) + return AdapterResult( + final_response=text, + stop_reason="end_turn", + run_status="completed", + response_emitted=True, + usage=state.usage, + side_call_failures=tuple(state.side_call_failures), + ) + + def _status_text(self, memory: MemoryWindow) -> str: + config = self._config + harness = self._harness + definitions = self._tool_registry.definitions() + reserve = _estimate_text_tokens(json.dumps(definitions)) + _OUTPUT_HEADROOM_TOKENS + used = sum(_estimate_tokens(message) for message in memory.window(reserve_tokens=reserve)) + reserve + if config.managed_mode: + # Study participants stay blind to their arm: no model, prompt + # profile, budgets, tools or switches. + percent = min(100, round(100 * used / max(1, memory.max_tokens))) + return "\n".join( + [ + "Session status", + f"- Context: about {percent}% of the conversation window in use", + f"- Undo checkpoints: {self._session_state.checkpoint_count()}", + ] + ) + switches = [ + name + for name in ( + "self_review", + "verify_on_stop", + "context_summarization", + "parallel_tools", + "project_instructions", + "read_before_edit", + "syntax_check", + "loop_guard", + "instruction_reminders", + "test_output_summary", + ) + if getattr(harness, name) + ] + instructions = self._project_instructions() + lines = [ + "Session status", + f"- Model: {config.adapter.provider.model or 'not configured'} (prompt profile: {self._prompt_profile()})", + f"- Step budget: {config.adapter.max_iterations} model calls per message; approval policy: {config.approval_policy}", + f"- Context: about {used:,} of {memory.max_tokens:,} tokens in use", + f"- Tools: {', '.join(sorted(self._tool_registry.known_tool_names())) or 'none'}", + f"- Commands: {self._commands_status()} " + f"(default timeout {int(config.commands.timeout_seconds)} s)", + f"- Verification: {_verification_label(harness, self._verify_command(), can_run=self._can_run_commands())}", + f"- Enabled behaviours: {', '.join(switches) or 'none'}", + f"- Project instructions: {', '.join(name for name, _chars in instructions.files) if instructions else 'none'}", + f"- Undo checkpoints: {self._session_state.checkpoint_count()}", + ] + return "\n".join(lines) + + def _undo_last_turn(self, memory: MemoryWindow) -> str: + outcome = self._session_state.undo_last_turn(self._tool_registry.file_tools) + if outcome.nothing_to_undo: + return "There are no file changes from this session to undo." + lines = ["Undid the file changes of the last turn that changed files."] + if outcome.restored: + lines.append(f"- Restored: {', '.join(outcome.restored)}") + if outcome.deleted: + lines.append(f"- Removed (created by the agent): {', '.join(outcome.deleted)}") + for path, reason in outcome.skipped: + lines.append(f"- Not changed: {path} ({reason})") + text = "\n".join(lines) + # Kept as the user's command and its result, so the model knows the + # files were reverted and a reopened chat replays both. + memory.append({"role": "user", "content": "/undo"}) + memory.append({"role": "assistant", "content": text}) + return text + + def _compact_now( + self, + memory: MemoryWindow, + state: _TurnState, + *, + run_id: str, + request_id: str, + cancellation_event: Event | None, + ) -> str | None: + definitions = self._tool_registry.definitions() + reserve = _estimate_text_tokens(json.dumps(definitions)) + _OUTPUT_HEADROOM_TOKENS + before = memory.estimated_tokens() + outcome = self._maybe_compact( + self._provider(), + memory, + state, + reserve_tokens=reserve, + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + keep_recent_units=2, + force=True, + ) + if outcome == "cancelled": + return None + if outcome == "failed": + return "Could not summarise the conversation (the model call failed); nothing was changed." + if outcome != "compacted": + return "Nothing to compact yet: the conversation is still short." + return ( + f"Summarised the earlier conversation into a checkpoint " + f"(about {before:,} → {memory.estimated_tokens():,} tokens of history)." + ) + + def _review_session( + self, + memory: MemoryWindow, + state: _TurnState, + *, + focus: str, + run_id: str, + request_id: str, + cancellation_event: Event | None, + ) -> str | None: + diff = self._session_state.session_diff(max_chars=_REVIEW_DIFF_CHARS) + if not diff.strip(): + return "There are no changes from this session to review." + request = "Review all changes made in this session." + if focus: + request += f" Focus: {focus}" + try: + issues = self._self_review( + self._provider(), + state, + prompt=request, + answer="(review requested by the user)", + diff=diff, + run_id=run_id, + request_id=request_id, + cancellation_event=cancellation_event, + context_budget=memory.max_tokens, + ) + except ProviderCancelled: + return None + text = ( + f"Review of this session's changes found possible issues:\n{issues}" + if issues + else "Review of this session's changes found no important issues." + ) + memory.append({"role": "user", "content": "/review" + (f" {focus}" if focus else "")}) + memory.append({"role": "assistant", "content": text}) + return text + + def _parse_output( + self, + turn: ProviderTurn, + *, + run_id: str, + request_id: str, + ) -> ParsedProviderOutput | None: + raw_output = turn.output.get("output", turn.output) if isinstance(turn.output, dict) else None + if not isinstance(raw_output, dict): + self._record_parse_failure( + run_id=run_id, + request_id=request_id, + output={"output": repr(turn.output)[:500]}, + failure_reason="missing_structured_output", + ) + return None + tool_calls_data = raw_output.get("tool_calls") + text = raw_output.get("final_answer", raw_output.get("text")) + if not isinstance(tool_calls_data, list) and not isinstance(text, str): + self._record_parse_failure( + run_id=run_id, + request_id=request_id, + output=raw_output, + failure_reason="missing_structured_output", + ) + return None + return ParsedProviderOutput( + text=_normalize_optional_text(text), + tool_calls=_normalize_tool_calls(tool_calls_data), + thought=_normalize_optional_text(raw_output.get("reasoning")), + finish_reason=turn.finish_reason, + reasoning_fields=_reasoning_fields(raw_output.get("reasoning_fields")), + ) + + # --------------------------------------------------------- telemetry + + def _record_parse_failure( + self, + *, + run_id: str, + request_id: str, + output: dict[str, Any], + failure_reason: str, + ) -> None: + self._telemetry.record( + event_type="agent.adapter.parse_failed", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "adapter_name": self._config.adapter.name, + "failure_reason": failure_reason, + }, + raw_payload=output, + ) + + def _record_loop_failure( + self, + *, + run_id: str, + request_id: str, + failure_reason: str, + extra_payload: dict[str, Any] | None = None, + metrics: dict[str, Any] | None = None, + ) -> None: + self._telemetry.record( + event_type="agent.adapter.loop_failed", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "adapter_name": self._config.adapter.name, + "max_iterations": self._config.adapter.max_iterations, + "failure_reason": failure_reason, + **(extra_payload or {}), + }, + metrics=metrics, + ) + + def _record_tool_failure( + self, + *, + run_id: str, + request_id: str, + tool_call: ToolCall, + failure_reason: str, + error_message: str, + ) -> None: + self._telemetry.record( + event_type="agent.tool.failed", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + "failure_reason": failure_reason, + "error_message": error_message, + }, + ) + + def _record_tool_denial( + self, + *, + run_id: str, + request_id: str, + tool_call: ToolCall, + denial_reason: str, + error_message: str, + ) -> None: + self._telemetry.record( + event_type="agent.tool.denied", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + "status": "denied", + "denial_reason": denial_reason, + "error_message": error_message, + }, + ) + + def _record_model_completed( + self, + *, + run_id: str, + request_id: str, + duration_ms: float, + provider_turn: ProviderTurn, + purpose: str = "turn", + ) -> None: + usage = provider_turn.usage + self._telemetry.record( + event_type="agent.model.completed", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "model": provider_turn.model, + "base_url": self._config.adapter.provider.base_url, + "finish_reason": provider_turn.finish_reason, + "usage": usage, + "usage_estimated": provider_turn.usage_estimated, + "call_purpose": purpose, + }, + metrics={ + "duration_ms": round(duration_ms, 3), + "prompt_tokens": usage.get("prompt_tokens", 0), + "completion_tokens": usage.get("completion_tokens", 0), + "total_tokens": usage.get("total_tokens", 0), + }, + raw_payload={ + "request": provider_turn.request_payload, + "response": provider_turn.raw_response, + }, + ) + + +def create_agent_adapter( + config: AgentConfig, + *, + telemetry: AgentTelemetryRecorder, + file_tools: WorkspaceFileTools, + command_tools: WorkspaceCommandTools, + event_sink: AgentEventSink | None = None, + mcp_tools: StdioMcpToolBroker | None = None, + session_state: SessionToolState | None = None, +) -> AgentAdapter: + if config.adapter.name == "openai_compatible_react": + return OpenAICompatibleReactAdapter( + config, + telemetry=telemetry, + tool_registry=ToolRegistry( + file_tools, + command_tools, + allowed_tools=config.allowed_tools + if config.allowed_tools is not None + else ( + frozenset(config.tools) + if getattr(config, "tools", None) is not None + else None + ), + approval_policy=config.approval_policy, + event_sink=event_sink, + mcp_tools=mcp_tools, + telemetry=telemetry, + workspace_root=config.workspace_root, + session_state=session_state, + harness=getattr(config, "harness", None), + ), + event_sink=event_sink, + session_state=session_state, + ) + return DeterministicEchoAdapter() + + +# ------------------------------------------------------- loop helpers + + +def _assistant_tool_call_message( + tool_calls: list[ToolCall], *, text: str | None, runtime: str | None = None +) -> dict[str, Any]: + message: dict[str, Any] = { + "role": "assistant", + "content": text or "", + "tool_calls": [ + { + "id": tool_call.tool_call_id, + "name": tool_call.name, + "arguments": dict(tool_call.arguments), + } + for tool_call in tool_calls + ], + } + if runtime is not None: + # Issued by the runtime itself (verify-on-stop), not chosen by the model. + message["code4me_runtime"] = runtime + return message + + +def _verification_label(harness: HarnessOptions, command: list[str] | None, *, can_run: bool) -> str: + """``command``: the verify argv this session will actually run (``_verify_command``).""" + if not harness.verify_on_stop: + return "off" + if not can_run: + return "off: commands cannot run in this session" + if command: + return f"the runtime runs {' '.join(command)} after changes" + if harness.verify_command: + return ( + "the agent is asked to verify after changes " + f"({' '.join(harness.verify_command)} cannot run in this session)" + ) + return "the agent is asked to verify after changes" + + +def _asked_question(tool_calls: list[ToolCall], results: list[dict[str, Any]]) -> str | None: + """The ask_user question of this batch, formatted as the turn's final message.""" + for call, result in zip(tool_calls, results): + if call.name != "ask_user" or result.get("status") != "ok": + continue + question = str(result.get("question") or "").strip() + options = [str(option) for option in result.get("options") or [] if str(option).strip()] + if not question: + continue + if options: + listed = "\n".join(f"{number}. {option}" for number, option in enumerate(options, start=1)) + return f"{question}\n\n{listed}" + return question + return None + + +def _review_issues(text: str | None) -> str | None: + """The reviewer's issue bullets, or None. + + The reviewer is asked for NO_ISSUES or '- ' bullets; a reply without + bullets (NO_ISSUES, or prose that ignored the format) is not actionable + and must not cost the agent a fix iteration. + """ + if not text: + return None + bullets = [line.strip() for line in text.splitlines() if re.match(r"^\s*[-*•]\s+\S", line)] + if not bullets: + return None + return "\n".join(bullets[:5])[:2000] + + +def _render_units_for_summary(units: list[list[dict[str, Any]]], *, max_chars: int) -> str: + """A compact transcript of the units to summarise, newest content kept when capped.""" + lines: list[str] = [] + for unit in units: + for message in unit: + role = message.get("role") + content = str(message.get("content") or "") + if role == "tool": + lines.append(f"TOOL RESULT ({message.get('name', 'tool')}): {content[:1500]}") + elif role == "assistant" and message.get("tool_calls"): + if content.strip(): + lines.append(f"ASSISTANT: {content[:2000]}") + for call in message.get("tool_calls") or []: + if isinstance(call, dict): + arguments = json.dumps(call.get("arguments", {}), sort_keys=True, default=str) + lines.append(f"TOOL CALL {call.get('name')}: {arguments[:600]}") + elif role == "user" and message.get("code4me_runtime") == "checkpoint": + lines.append(f"EARLIER CHECKPOINT:\n{content[:6000]}") + elif role == "user": + lines.append(f"USER: {content[:4000]}") + else: + lines.append(f"ASSISTANT: {content[:3000]}") + text = "\n".join(lines) + if len(text) > max_chars: + text = "[earliest part omitted]\n" + text[-max_chars:] + return text + + +def _tool_message(tool_call: ToolCall, result: dict[str, Any]) -> dict[str, Any]: + return { + "role": "tool", + "tool_call_id": tool_call.tool_call_id, + "name": tool_call.name, + "content": json.dumps(result, sort_keys=True, default=str), + } + + +def _not_run_result(tool_call: ToolCall, reason: str) -> dict[str, Any]: + return { + "status": "skipped", + "reason": reason, + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + } + + +def _cancelled_tool_result(tool_call: ToolCall) -> dict[str, Any]: + return { + "status": "cancelled", + "error": "Cancelled by user before execution", + "tool_name": tool_call.name, + "tool_call_id": tool_call.tool_call_id, + } + + +def _finalize_tool_output(output: dict[str, Any], base: dict[str, Any]) -> dict[str, Any]: + result = dict(output) + tool_status = str(result.get("status", "")).lower() + if result.get("timed_out") or tool_status == "timeout": + status = "timeout" + elif tool_status == "cancelled": + status = "cancelled" + elif tool_status in {"failed", "error"}: + status = "error" + else: + status = "ok" + if tool_status and tool_status not in {"ok", status}: + result["tool_status"] = result["status"] + result.update(base) + result["status"] = status + return result + + +_UNCAPPED_RESULT_KEYS = frozenset( + {"tool_call_id", "tool_name", "status", "error_code", "reason", "path", "tool_status"} +) +_NUMBERED_LINE_RE = re.compile(r"^(\d+)\|") + + +def _cap_tool_result(result: dict[str, Any], max_chars: int) -> dict[str, Any]: + def size(value: dict[str, Any]) -> int: + return len(json.dumps(value, sort_keys=True, default=str)) + + if size(result) <= max_chars: + return result + capped = dict(result) + for _ in range(8): + overflow = size(capped) - max_chars + if overflow <= 0: + break + key = max( + (k for k, v in capped.items() if isinstance(v, str) and k not in _UNCAPPED_RESULT_KEYS), + key=lambda k: len(capped[k]), + default=None, + ) + if key is None or len(capped[key]) < 200: + list_key = max( + (k for k, v in capped.items() if isinstance(v, list)), + key=lambda k: len(json.dumps(capped[k], default=str)), + default=None, + ) + if list_key is None or not capped[list_key]: + break + items = capped[list_key] + capped[list_key] = items[: max(1, len(items) // 2)] + capped["truncated"] = True + continue + keep = max(100, len(capped[key]) - overflow - 64) + if key == "content" and "end_line" in capped: + _cap_numbered_content(capped, keep) + else: + omitted = len(capped[key]) - keep + capped[key] = capped[key][:keep] + f"…[truncated {omitted} chars]" + capped["truncated"] = True + return capped + + +def _cap_numbered_content(capped: dict[str, Any], keep: int) -> None: + """Cut read_file output at a line boundary and keep its paging fields honest.""" + content = str(capped["content"]) + head = content[:keep] + lines = head.split("\n") + complete = lines[:-1] if len(lines) > 1 else [] + last_line = None + for line in reversed(complete): + match = _NUMBERED_LINE_RE.match(line) + if match: + last_line = int(match.group(1)) + break + if last_line is None: + omitted = len(content) - keep + capped["content"] = head + f"…[truncated {omitted} chars]" + return + kept_lines = [line for line in complete if not line.startswith("[truncated:")] + next_offset = last_line + 1 + total = capped.get("total_lines") + capped["content"] = "\n".join(kept_lines) + ( + f"\n[truncated to fit the context budget: showing lines {capped.get('start_line', 1)}-" + f"{last_line}{f' of {total}' if isinstance(total, int) else ''}; call read_file with " + f"offset={next_offset} to continue]" + ) + capped["end_line"] = last_line + capped["next_offset"] = next_offset + capped["truncation_reason"] = "context_budget" + + +def _batch_result_cap(max_tokens: int, batch_size: int) -> int: + """Characters allowed per tool result so a whole batch fits half the context budget.""" + per_result = (max(1, int(max_tokens)) * 4) // (2 * max(1, batch_size)) + return int(min(_MAX_TOOL_RESULT_CHARS, max(4_000, per_result))) + + +def _budget_exhausted_text(tool_calls: list[ToolCall], results: list[dict[str, Any]]) -> str: + lines = [ + "I used the step budget for this message, so I could not review these results; here is " + "what ran on the last step:" + ] + for tool_call, result in zip(tool_calls, results): + summary = _tool_result_summary(tool_call.name, result) or result.get("status", "done") + lines.append(f"- {tool_call.name}: {result.get('status', 'ok')} — {summary}") + lines.append("Send another message to continue.") + return "\n".join(lines) + + +def _stop_reason_from_finish(finish_reason: str | None) -> str: + if finish_reason == "length": + return "max_tokens" + if finish_reason == "content_filter": + return "refusal" + return "end_turn" + + +def _model_failure_details( + exc: BaseException, *, started_at: float, purpose: str +) -> tuple[dict[str, Any], dict[str, Any]]: + """(payload, metrics) describing a failed model call. + + Numbers go into metrics, which the server keeps as structural data even + without content consent; ``error_message`` is a content key. + """ + payload: dict[str, Any] = { + "call_purpose": purpose, + "error_type": type(exc).__name__, + "error_message": str(exc)[:500], + } + metrics: dict[str, Any] = {"duration_ms": round((perf_counter() - started_at) * 1000, 3)} + for key in ("status_code", "attempts"): + value = getattr(exc, key, None) + if isinstance(value, int) and not isinstance(value, bool): + metrics[key] = value + return payload, metrics + + +def _provider_error_text(exc: ProviderRequestFailed) -> str: + if exc.status_code: + return ( + f"I couldn't reach the model service (HTTP {exc.status_code} after {exc.attempts} " + f"attempt{'s' if exc.attempts != 1 else ''}). Nothing further was changed; please send " + "your message again." + ) + return ( + f"I couldn't reach the model service ({exc}; {exc.attempts} attempt" + f"{'s' if exc.attempts != 1 else ''}). Nothing further was changed; please send your " + "message again." + ) + + +def _hint_for(tool_name: str, exc: BaseException) -> str: + code = getattr(exc, "code", None) + if isinstance(exc, (ToolFileNotFoundError, FileNotFoundError)) or code == "file_not_found": + return "Check the path with glob_files or list_files; paths are workspace-relative." + if code in {"edit_no_match", "edit_ambiguous"} or ( + isinstance(exc, ValueError) and tool_name in {"replace_text", "edit_file"} + ): + return "old_text must match the file exactly (including whitespace); read_file the region and retry with the exact text." + if isinstance(exc, FileExistsError) or code == "file_exists": + return "Use write_file or edit_file for an existing file, or choose a different path." + if isinstance(exc, PermissionError) or code == "outside_workspace": + return "Stay inside the workspace, and do not run a program the study blocks." + if code == "not_text_file": + if tool_name == "write_file": + return ( + "The existing file is not UTF-8 text, so it is not overwritten blindly; delete it " + "with delete_file first if you really want to replace it." + ) + return "This tool only works with UTF-8 text files." + if code == "command_not_found": + return "Use a program that is installed on this machine, or the path of a script in the project." + if isinstance(exc, (KeyError, TypeError)): + return f"Required argument missing or of the wrong type: {exc}." + return "Adjust the arguments or try a different approach." + + +def _tool_call_id(raw_id: Any) -> str: + """The provider's call id, or a fresh one when it sends none (or null/blank). + + A step counter would repeat every step, so telemetry and traces would merge + the calls of different steps into one. + """ + text = str(raw_id).strip() if raw_id is not None else "" + return text or f"call-{uuid.uuid4().hex}" + + +def _normalize_tool_calls(value: Any) -> list[ToolCall]: + if not isinstance(value, list): + return [] + normalized: list[ToolCall] = [] + for raw_tool_call in value: + if not isinstance(raw_tool_call, dict): + continue + arguments = raw_tool_call.get("arguments", {}) + if arguments is None: + arguments = {} + argument_error = raw_tool_call.get("argument_error") + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError as exc: + argument_error = argument_error or f"Tool arguments were not valid JSON: {exc}" + arguments = {} + if not isinstance(arguments, dict): + if argument_error is None: + argument_error = "Tool arguments must be a JSON object." + arguments = {} + name = str(raw_tool_call.get("name", "")).strip() + if not name: + continue + normalized.append( + ToolCall( + tool_call_id=_tool_call_id(raw_tool_call.get("id")), + name=name, + arguments=dict(arguments), + argument_error=str(argument_error) if argument_error else None, + raw_arguments=_normalize_optional_text(raw_tool_call.get("raw_arguments")), + ) + ) + return normalized + + +def _normalize_optional_text(value: Any) -> str | None: + if not isinstance(value, str): + return None + normalized = value.strip() + return normalized or None + + +# ------------------------------------------------------------ tool cards + + +def _absolute_path(workspace_root: Path | None, path: str | None) -> str | None: + if not path: + return None + if workspace_root is None: + return path + candidate = Path(path).expanduser() + if candidate.is_absolute(): + return os.path.normpath(str(candidate)) + return os.path.normpath(os.path.join(str(workspace_root), path)) + + +def _filename(path: str) -> str: + return path.rstrip("/").rsplit("/", 1)[-1] or path + + +def _safe_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _safe_len(value: Any) -> int: + try: + return len(value) + except TypeError: + return 0 + + +def _safe_tool_event_metadata( + tool_name: str, + values: dict[str, Any], + *, + workspace_root: Path | None = None, +) -> dict[str, Any] | None: + """Card metadata that never raises, even for arguments that failed validation.""" + try: + return _tool_event_metadata(tool_name, values, workspace_root=workspace_root) + except Exception: # noqa: BLE001 + if tool_name in tool_catalog.NO_CARD_TOOLS: + return None + return { + "kind": tool_kind(tool_name), + "title": f"Run {tool_name}", + "content_text": f"Run {tool_name}", + "raw_input": None, + } + + +def _tool_event_metadata( + tool_name: str, + values: dict[str, Any], + *, + workspace_root: Path | None = None, +) -> dict[str, Any] | None: + kind = tool_kind(tool_name) + if tool_name in tool_catalog.NO_CARD_TOOLS: + return None + if tool_name == "apply_patch": + actions = values.get("_patch") or [] + targets = [ + str(getattr(action, "move_to", None) or getattr(action, "path", "")) + for action in actions + ] + targets = [target for target in targets if target] + absolute = tuple( + item for item in (_absolute_path(workspace_root, target) for target in targets) if item + ) + if not targets: + title = "Apply patch" + elif len(targets) == 1: + title = f"Apply patch to {targets[0]}" + else: + title = f"Apply patch to {targets[0]} and {len(targets) - 1} more file{'s' if len(targets) > 2 else ''}" + return { + "kind": kind, + "title": title, + "path": absolute[0] if absolute else None, + "locations": absolute or None, + "content_text": title, + "raw_input": {"files": targets, "file_count": len(targets)}, + } + if tool_name == "read_file": + path = str(values.get("path", "")).strip() + title = f"Read {path}" if path else "Read file" + offset = _safe_int(values.get("offset")) or _safe_int(values.get("line_start")) + limit = _safe_int(values.get("limit")) + if offset and limit: + title += f" (lines {offset}-{offset + limit - 1})" + elif offset: + title += f" (from line {offset})" + raw_input: dict[str, Any] = {"path": path} if path else {} + if offset: + raw_input["offset"] = offset + if limit: + raw_input["limit"] = limit + return { + "kind": kind, + "title": title, + "path": _absolute_path(workspace_root, path), + "content_text": f"Read file {_filename(path)}" if path else "Read file", + "raw_input": raw_input or None, + } + if tool_name in {"create_file", "write_file", "replace_text", "edit_file", "delete_file"}: + path = str(values.get("path", "")).strip() + prefixes = { + "create_file": ("Create", "Created file"), + "write_file": ("Update", "Updated file"), + "replace_text": ("Replace text in", "Replaced text in"), + "edit_file": ("Edit", "Edited"), + "delete_file": ("Delete", "Deleted"), + } + title_prefix, content_prefix = prefixes[tool_name] + title = f"{title_prefix} {path}" if path else title_prefix + raw_input = {"path": path} if path else {} + if tool_name == "edit_file": + edit_count = _safe_len(values.get("edits")) + title += f" ({edit_count} edit{'s' if edit_count != 1 else ''})" + raw_input["edit_count"] = edit_count + if tool_name == "replace_text" and values.get("replace_all"): + raw_input["replace_all"] = True + return { + "kind": kind, + "title": title, + "path": _absolute_path(workspace_root, path), + "content_text": f"{content_prefix} {_filename(path)}" if path else content_prefix, + "raw_input": raw_input or None, + } + if tool_name == "move_file": + source = str(values.get("source_path", "")).strip() + destination = str(values.get("destination_path", "")).strip() + source_abs = _absolute_path(workspace_root, source) + destination_abs = _absolute_path(workspace_root, destination) + locations = tuple(item for item in (source_abs, destination_abs) if item) + return { + "kind": kind, + "title": f"Move {source} → {destination}" if source and destination else "Move file", + "path": source_abs, + "locations": locations or None, + "content_text": f"Moved {source} to {destination}", + "raw_input": { + "source_path": source, + "destination_path": destination, + "overwrite": bool(values.get("overwrite", False)), + }, + } + if tool_name == "list_files": + path = str(values.get("path", ".") or ".").strip() + return { + "kind": kind, + "title": f"List {path}", + "path": _absolute_path(workspace_root, path), + "content_text": f"Listed {path}", + "raw_input": {"path": path, "recursive": bool(values.get("recursive", False))}, + } + if tool_name == "glob_files": + pattern = str(values.get("pattern", "")).strip() + path = str(values.get("path", ".") or ".").strip() + title = f"Find files matching {pattern}" if pattern else "Find files" + if path and path != ".": + title += f" in {path}" + return { + "kind": kind, + "title": title, + "path": _absolute_path(workspace_root, path), + "content_text": title, + "raw_input": {"pattern": pattern, "path": path}, + } + if tool_name in {"grep_files", "search_files"}: + pattern = str(values.get("pattern", values.get("query", ""))).strip() + path = str(values.get("path", ".") or ".").strip() + title = f'Search "{pattern}"' if pattern else "Search files" + if path and path != ".": + title += f" in {path}" + raw_input = {"pattern": pattern, "path": path} + if values.get("glob"): + raw_input["glob"] = values["glob"] + if values.get("output_mode"): + raw_input["output_mode"] = values["output_mode"] + return { + "kind": kind, + "title": title, + "path": _absolute_path(workspace_root, path), + "content_text": title, + "raw_input": raw_input, + } + if tool_name == "run_command": + argv = values.get("argv") + command = " ".join(str(arg) for arg in argv) if isinstance(argv, (list, tuple)) else "command" + raw_input: dict[str, Any] = {"argv": argv, "cwd": values.get("cwd", ".")} + if values.get("timeout_seconds") is not None: + raw_input["timeout_seconds"] = values["timeout_seconds"] + return { + "kind": kind, + "title": f"Run {command}", + "path": None, + "content_text": f"Run command: {command}", + "raw_input": raw_input, + } + if tool_name.startswith("mcp__"): + parts = tool_name.split("__", 2) + if len(parts) == 3: + title = f"Call {parts[1]}: {parts[2]}" + else: + title = f"Call MCP tool {tool_name.removeprefix('mcp__')}" + return { + "kind": "other", + "title": title, + "content_text": title, + "raw_input": values, + } + return { + "kind": kind, + "title": f"Run tool {tool_name}", + "content_text": f"Run tool {tool_name}", + "raw_input": values, + } + + +def _tool_result_summary(tool_name: str, output: dict[str, Any]) -> str | None: + try: + if tool_name == "read_file": + if "total_lines" in output: + start = int(output.get("start_line") or 0) + end = int(output.get("end_line") or 0) + count = max(0, end - start + 1) if end >= start and start > 0 else 0 + text = f"Read {count} line{'s' if count != 1 else ''} of {_filename(str(output.get('path', '')))}" + if output.get("truncated"): + text += f" (of {output['total_lines']}; truncated)" + return text + return None + if tool_name == "list_files": + entries = output.get("entries", output.get("paths")) + if isinstance(entries, list): + text = f"Listed {len(entries)} entr{'ies' if len(entries) != 1 else 'y'}" + return text + (" (truncated)" if output.get("truncated") else "") + return None + if tool_name == "glob_files": + files = output.get("files") + if isinstance(files, list): + text = f"Found {len(files)} file{'s' if len(files) != 1 else ''}" + return text + (" (truncated)" if output.get("truncated") else "") + return None + if tool_name in {"grep_files", "search_files"}: + if "match_count" in output: + matches = int(output.get("match_count") or 0) + files = int(output.get("file_count") or 0) + text = f"Found {matches} match{'es' if matches != 1 else ''} in {files} file{'s' if files != 1 else ''}" + return text + (" (truncated)" if output.get("truncated") else "") + matches = output.get("matches") + if isinstance(matches, list): + return f"Found {len(matches)} match{'es' if len(matches) != 1 else ''}" + return None + if tool_name == "run_command": + if output.get("timed_out"): + text = f"Timed out after {output.get('timeout_seconds', '?')} s" + elif output.get("status") == "cancelled": + text = "Cancelled" + elif "exit_code" in output: + duration = output.get("duration_ms") + text = f"Exit code {output.get('exit_code')}" + if isinstance(duration, (int, float)): + text += f" in {duration / 1000:.1f} s" + else: + return None + test_summary = output.get("test_summary") + if isinstance(test_summary, dict): + headline = _test_headline(test_summary) + if headline: + text = f"{text}\n{headline}" + tail = _output_tail(output) + return f"{text}\n{tail}" if tail else text + if tool_name == "apply_patch": + files = output.get("files") + if isinstance(files, list): + return "; ".join( + f"{item.get('action')} {item.get('move_to') or item.get('path')}" + for item in files + if isinstance(item, dict) + ) + return None + if tool_name == "replace_text": + if "replacements" in output: + count = int(output["replacements"]) + return f"Replaced {count} occurrence{'s' if count != 1 else ''}" + return None + if tool_name == "edit_file": + if "edits_applied" in output: + count = int(output["edits_applied"]) + return f"Applied {count} edit{'s' if count != 1 else ''}" + return None + if tool_name == "delete_file": + return f"Deleted {output.get('path', 'file')}" + if tool_name == "move_file": + return f"Moved {output.get('source_path', '')} to {output.get('destination_path', '')}" + if tool_name.startswith("mcp__"): + status = output.get("status") + return f"MCP tool {status}" if status else None + except (TypeError, ValueError): + return None + return None + + +def _test_headline(test_summary: dict[str, Any]) -> str | None: + counts = [ + f"{test_summary[key]} {key}" + for key in ("failed", "errors", "passed", "skipped") + if isinstance(test_summary.get(key), int) and test_summary[key] + ] + framework = test_summary.get("framework") or "tests" + text = f"{framework}: {', '.join(counts)}" if counts else None + failures = test_summary.get("failures") + if isinstance(failures, list) and failures: + names = "; ".join(str(item.get("name")) for item in failures[:5] if isinstance(item, dict)) + text = f"{text or framework}\nFailing: {names}" + return text + + +def _result_notes(tool_name: str, output: dict[str, Any]) -> list[str]: + """Warnings the model and the user should notice next to a completed edit.""" + notes: list[str] = [] + strategies = output.get("match_strategies") + if isinstance(strategies, list) and strategies: + fuzzy = sorted({describe_strategy(item) for item in strategies if item not in ("exact", "line_endings")}) + if fuzzy: + notes.append(f"Matched {', '.join(fuzzy)}; check the diff.") + syntax = output.get("syntax_error") + problems = [syntax] if isinstance(syntax, dict) else list(output.get("syntax_errors") or []) + for problem in problems: + if isinstance(problem, dict): + where = f"{problem.get('path')}:" if problem.get("path") else "line " + notes.append( + f"Syntax error introduced ({problem.get('language')}, {where}{problem.get('line')}): " + f"{problem.get('message')}" + ) + return notes + + +def _completed_card_text( + tool_name: str, + output: dict[str, Any], + metadata: dict[str, Any] | None, + has_diff: bool, +) -> str | None: + notes = _result_notes(tool_name, output) + if has_diff: + return "\n".join(notes) or None + summary = _tool_result_summary(tool_name, output) or (metadata or {}).get("content_text") + parts = [summary, *notes] if summary else notes + return "\n".join(parts) or None + + +def _output_tail(output: dict[str, Any], max_lines: int = 40, max_chars: int = 2000) -> str: + parts: list[str] = [] + for key in ("stdout", "stderr"): + value = output.get(key) + if isinstance(value, str) and value.strip(): + lines = value.rstrip().splitlines()[-max_lines:] + text = "\n".join(lines) + if len(text) > max_chars: + text = text[-max_chars:] + parts.append(text if key == "stdout" else f"[stderr]\n{text}") + return "\n".join(parts) + + +# ------------------------------------------------- provider wire format + + +def _tool_definitions() -> list[dict[str, Any]]: + return tool_catalog.tool_definitions() + + +def _function_tool( + name: str, + description: str, + properties: dict[str, Any], + required: list[str], +) -> dict[str, Any]: + return tool_catalog._function_tool(name, description, properties, required) + + +# Reasoning is sent back as one field, ``reasoning_content``: the name DeepSeek, +# Kimi, GLM and Qwen require and vLLM, SGLang and OpenRouter (as an alias) +# accept, normalised from ``reasoning`` as the AI SDK's openai-compatible +# provider does. OpenRouter's structured ``reasoning_details`` is not echoed. +# Servers that reject the field (Groq, Mistral, TensorRT-LLM) are handled by +# the provider's one-time fallback; our relay strips it before any upstream. +_REASONING_FIELD = "reasoning_content" + + +# Programs that inspect or move files but never run the code: after an edit, +# running only these is not verification (git status/diff was the last command +# in 44 of 68 benchmark tasks that "verified"). +_NON_VERIFYING_PROGRAMS = frozenset({ + "git", "ls", "cat", "head", "tail", "grep", "rg", "find", "wc", "echo", "pwd", "which", + "type", "file", "stat", "tree", "du", "df", "sort", "uniq", "diff", "cut", "tr", "less", + "more", "env", "printenv", "true", "false", "date", "whoami", "id", "uname", "basename", + "dirname", "realpath", "readlink", "nl", "sed", "awk", "rm", "mv", "cp", "mkdir", "touch", + "chmod", "ln", "cd", "export", "set", "unset", +}) +_SCRIPT_SEPARATORS = re.compile(r"&&|\|\||[;|\n]") + + +_INSTALL_SUBCOMMANDS = frozenset({"install", "i", "add", "ci", "uninstall", "remove", "sync", "update"}) + + +def _is_install_command(program: str, args: list[str]) -> bool: + """Package management (pip/uv/npm install…): changes the environment, checks nothing.""" + if program in {"pip", "pip3", "conda", "mamba", "apt", "apt-get", "brew"}: + return True + if program in {"python", "python3", "py"} and args[:2] == ["-m", "pip"]: + return True + if program == "uv": + return not args or args[0] != "run" + if program in {"npm", "yarn", "pnpm", "bun"}: + return bool(args) and args[0] in _INSTALL_SUBCOMMANDS + return False + + +def _is_verification_command(argv: list[object]) -> bool: + """Whether a command run counts as checking the code (verify-on-stop). + + Shell scripts (``bash -c "cd src && pytest -q"``) count when any segment + runs something other than inspection; an unparsable script counts, so the + gate never nudges because of our own parsing limits. + """ + if not argv: + return False + program = PurePath(str(argv[0])).name.lower() + if program.endswith((".exe", ".bat", ".cmd")): + program = program.rsplit(".", 1)[0] + if program in _SHELL_PROGRAMS and len(argv) >= 3 and str(argv[1]).startswith("-") and "c" in str(argv[1]): + script = str(argv[2]) + for segment in _SCRIPT_SEPARATORS.split(script): + try: + words = shlex.split(segment) + except ValueError: + return True + while words and re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", words[0]): + words = words[1:] # VAR=value prefixes + if words and _is_verification_command(words): + return True + return False + if _is_install_command(program, [str(arg) for arg in argv[1:]]): + return False + return program not in _NON_VERIFYING_PROGRAMS + + +_MORE_WORK_START = re.compile( + r"^(?:(?:now|next|first|then|also|so|ok(?:ay)?),?\s+)?" + r"(?:let me|let's|let us|i'll|i will|i'm going to|i am going to)\b", + re.IGNORECASE, +) + + +def _announces_more_work(text: str) -> bool: + """A final message that ends by announcing a next step it did not take. + + Checked on the last sentence only; "Let me know…" closes a turn politely and + does not count. + """ + stripped = text.strip() + if not stripped: + return False + if stripped.endswith(":"): + return True + last = re.split(r"(?<=[.!?])\s+|\n+", stripped)[-1].strip() + if re.search(r"\blet me know\b", last, re.IGNORECASE): + return False + return bool(_MORE_WORK_START.match(last)) + + +def _reasoning_fields(source: object) -> dict[str, Any] | None: + """``{"reasoning_content": text}`` from a response message; exact text ("" kept).""" + if not isinstance(source, dict): + return None + for key in (_REASONING_FIELD, "reasoning"): + value = source.get(key) + if isinstance(value, str): + return {_REASONING_FIELD: value} + return None + + +def _with_reasoning(message: dict[str, Any], state: "_TurnState") -> dict[str, Any]: + """Attach the latest response's reasoning to the message holding its output. + + Consumed once, so runtime messages appended later in the same step never + carry it. DeepSeek thinking mode with tools requires it on later requests. + """ + if state.reasoning_fields: + message = {**message, "provider_reasoning": dict(state.reasoning_fields)} + state.reasoning_fields = None + return message + + +def _to_openai_message(message: dict[str, Any]) -> dict[str, Any]: + role = str(message.get("role", "user")) + converted = _to_openai_message_body(message, role) + if role == "assistant": + converted.update(_reasoning_fields(message.get("provider_reasoning")) or {}) + return converted + + +# DeepSeek V4 with tools: "the reasoning_content must be fully passed back to the +# API in all subsequent requests — even for turns where the model did not +# perform a tool call", else HTTP 400. The AI SDK and opencode fill "" on every +# assistant message for these model ids; so do we. +_DEEPSEEK_V4_MODEL = re.compile(r"(?:^|/)deepseek-(?:v4|flash|pro)", re.IGNORECASE) + + +def _needs_reasoning_on_every_assistant(model: str) -> bool: + return bool(_DEEPSEEK_V4_MODEL.search(model or "")) + + +def _apply_reasoning_policy(messages: list[dict[str, Any]], *, echo: bool, fill: bool) -> None: + """Strip or complete ``reasoning_content`` on outgoing assistant messages, in place.""" + for message in messages: + if message.get("role") != "assistant": + continue + if not echo: + message.pop(_REASONING_FIELD, None) + elif fill and not isinstance(message.get(_REASONING_FIELD), str): + message[_REASONING_FIELD] = "" + + +def _reasoning_rejection(exc: "ProviderRequestFailed") -> str | None: + """How a 400 treats reasoning: ``rejected`` (unknown field) or ``required``.""" + if exc.status_code != 400: + return None + text = str(exc).lower() + if "reasoning" not in text: + return None + if "passed back" in text: + return "required" + if any(word in text for word in ("extra", "not permitted", "unsupported", "unknown", "unrecognized", "not allowed")): + return "rejected" + return None + + +def _to_openai_message_body(message: dict[str, Any], role: str) -> dict[str, Any]: + if role == "assistant" and message.get("tool_calls"): + return { + "role": "assistant", + "content": message.get("content", ""), + "tool_calls": [ + { + "id": str(tool_call.get("id", "")), + "type": "function", + "function": { + "name": str(tool_call.get("name", "")), + "arguments": json.dumps( + tool_call.get("arguments", {}), sort_keys=True, default=str + ), + }, + } + for tool_call in message.get("tool_calls", []) + if isinstance(tool_call, dict) + ], + } + if role == "tool": + return { + "role": "tool", + "tool_call_id": str(message.get("tool_call_id", "")), + "content": str(message.get("content", "")), + } + return { + "role": role, + "content": str(message.get("content", "")), + } + + +def _normalize_openai_provider_response( + response_payload: dict[str, Any] +) -> dict[str, Any]: + choices = response_payload.get("choices") + if not isinstance(choices, list) or not choices: + raise RuntimeError("Provider response did not include choices.") + first_choice = choices[0] if isinstance(choices[0], dict) else {} + message = first_choice.get("message", {}) + if not isinstance(message, dict): + raise RuntimeError("Provider response choice message was not an object.") + + tool_calls = [] + raw_tool_calls = message.get("tool_calls", []) + if isinstance(raw_tool_calls, list): + for raw_tool_call in raw_tool_calls: + if not isinstance(raw_tool_call, dict): + continue + function_data = raw_tool_call.get("function", {}) + if not isinstance(function_data, dict): + continue + raw_arguments = function_data.get("arguments", "{}") + if raw_arguments is None: + raw_arguments = "{}" + argument_error: str | None = None + if isinstance(raw_arguments, dict): + arguments: dict[str, Any] = raw_arguments + raw_arguments_text = json.dumps(raw_arguments) + else: + raw_arguments_text = str(raw_arguments) + arguments = {} + try: + parsed = json.loads(raw_arguments_text or "{}") + except json.JSONDecodeError as exc: + argument_error = f"Tool arguments were not valid JSON: {exc}" + else: + if isinstance(parsed, dict): + arguments = parsed + else: + argument_error = "Tool arguments must be a JSON object." + name = str(function_data.get("name", "")).strip() + if not name: + continue + entry: dict[str, Any] = { + "id": _tool_call_id(raw_tool_call.get("id")), + "name": name, + "arguments": arguments, + } + if argument_error is not None: + entry["argument_error"] = argument_error + entry["raw_arguments"] = raw_arguments_text[:500] + tool_calls.append(entry) + + final_answer = message.get("content") + if isinstance(final_answer, list): + final_answer = "".join( + str(block.get("text", "")) + for block in final_answer + if isinstance(block, dict) and block.get("type") == "text" + ) + + normalized: dict[str, Any] = {"tool_calls": tool_calls} + reasoning = _normalize_optional_text( + message.get("reasoning") or message.get("reasoning_content") + ) + if reasoning is not None: + normalized["reasoning"] = reasoning + reasoning_fields = _reasoning_fields(message) + if reasoning_fields: + normalized["reasoning_fields"] = reasoning_fields + if final_answer is not None: + normalized["final_answer"] = str(final_answer) + return normalized + + +def _response_finish_reason(response_payload: dict[str, Any]) -> str | None: + choices = response_payload.get("choices") + if not isinstance(choices, list) or not choices: + return None + first_choice = choices[0] if isinstance(choices[0], dict) else {} + finish_reason = first_choice.get("finish_reason") + return str(finish_reason) if finish_reason is not None else None + + +def _usage_is_reported(raw_usage: Any) -> bool: + if not isinstance(raw_usage, dict): + return False + return any( + _int_or_zero(raw_usage.get(key)) > 0 + for key in ("prompt_tokens", "completion_tokens", "total_tokens") + ) + + +def _normalize_usage( + raw_usage: Any, + messages: list[dict[str, Any]], + output: dict[str, Any], +) -> dict[str, int]: + if isinstance(raw_usage, dict): + prompt_tokens = _int_or_zero(raw_usage.get("prompt_tokens")) + completion_tokens = _int_or_zero(raw_usage.get("completion_tokens")) + total_tokens = _int_or_zero(raw_usage.get("total_tokens")) + if total_tokens <= 0: + total_tokens = prompt_tokens + completion_tokens + if prompt_tokens > 0 or completion_tokens > 0 or total_tokens > 0: + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": total_tokens, + } + + prompt_tokens = sum(_estimate_tokens(message) for message in messages) + completion_tokens = _estimated_output_token_count(output) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + } + + +def _estimated_output_token_count(output: dict[str, Any]) -> int: + parts: list[str] = [] + final_answer = output.get("final_answer") + if final_answer: + parts.append(str(final_answer)) + tool_calls = output.get("tool_calls", []) + if isinstance(tool_calls, list) and tool_calls: + parts.append(json.dumps(tool_calls, sort_keys=True, default=str)) + text = " ".join(part for part in parts if part).strip() + if not text: + return 1 + return _estimate_text_tokens(text) + + +def _int_or_zero(value: Any) -> int: + try: + return max(0, int(value)) + except (TypeError, ValueError): + return 0 diff --git a/src/code4me2_agent/async_bridge.py b/src/code4me2_agent/async_bridge.py new file mode 100644 index 0000000..797e908 --- /dev/null +++ b/src/code4me2_agent/async_bridge.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import asyncio +import concurrent.futures +from queue import Queue +from threading import Event, Thread +from time import monotonic +from typing import Any + +_CANCEL_POLL_SECONDS = 0.25 + + +class OperationCancelled(RuntimeError): + """The blocking wait was abandoned because the session cancel event fired.""" + + +def run_awaitable_blocking(awaitable: Any, *, timeout_seconds: float | None = None) -> object: + wrapped = awaitable if timeout_seconds is None else asyncio.wait_for(awaitable, timeout_seconds) + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(wrapped) + return _run_awaitable_in_worker_thread(wrapped) + + +def _run_awaitable_in_worker_thread(awaitable: Any) -> object: + outcome: Queue[tuple[bool, object]] = Queue(maxsize=1) + + def worker() -> None: + try: + outcome.put((True, asyncio.run(awaitable))) + except BaseException as exc: # noqa: BLE001 + outcome.put((False, exc)) + + thread = Thread(target=worker, daemon=True) + thread.start() + thread.join() + succeeded, value = outcome.get() + if succeeded: + return value + raise value + + +class EventLoopAsyncRunner: + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + + def run( + self, + awaitable: Any, + *, + timeout_seconds: float | None = None, + cancel_event: Event | None = None, + ) -> object: + if self._loop.is_closed() or not self._loop.is_running(): + return run_awaitable_blocking(awaitable, timeout_seconds=timeout_seconds) + try: + running_loop = asyncio.get_running_loop() + except RuntimeError: + running_loop = None + if running_loop is self._loop: + raise RuntimeError("Cannot block on an ACP client call from the ACP event loop.") + wrapped = awaitable if timeout_seconds is None else asyncio.wait_for(awaitable, timeout_seconds) + future = asyncio.run_coroutine_threadsafe(wrapped, self._loop) + if cancel_event is None: + try: + return future.result(timeout=timeout_seconds) + except concurrent.futures.TimeoutError as exc: + future.cancel() + raise TimeoutError("Timed out waiting for ACP client response.") from exc + # Poll in short slices so a session cancel can abandon the wait; the + # pending JSON-RPC request is cancelled on the loop side. + deadline = None if timeout_seconds is None else monotonic() + timeout_seconds + 1.0 + while True: + try: + return future.result(timeout=_CANCEL_POLL_SECONDS) + except concurrent.futures.TimeoutError as exc: + if cancel_event.is_set(): + future.cancel() + raise OperationCancelled( + "Cancelled while waiting for the ACP client." + ) from None + if deadline is not None and monotonic() >= deadline: + future.cancel() + raise TimeoutError("Timed out waiting for ACP client response.") from exc diff --git a/src/code4me2_agent/cli.py b/src/code4me2_agent/cli.py new file mode 100644 index 0000000..75b30a9 --- /dev/null +++ b/src/code4me2_agent/cli.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import platform +import shutil +import sys +from pathlib import Path + +from code4me2_agent._build_version import RUNTIME_VERSION +from code4me2_agent.acp_runtime import run_acp_stdio +from code4me2_agent.config import ( + AdapterConfig, + AgentConfig, + CommandConfig, + MemoryWindowConfig, + OpenAICompatibleProviderConfig, +) + +DEFAULT_CONFIG_PATH = os.path.expanduser("~/.code4me/agent-config.json") +DEFAULT_ACP_JSON_PATH = os.path.expanduser("~/.code4me/acp.json") +SUPPORTED_MANAGED_PLATFORMS = {"darwin", "linux", "win32"} + +# Scaffold written by `code4me2-agent init`. Deliberately specifies no model, +# provider or API key: those come from the agent profile the backend assigns +# (GET /api/acp/agent-config), so hardcoding them here would let a stale local +# file silently override a study assignment. +# +# `max_iterations` and the blocked commands are also server-overridable — the +# values here only apply before the first successful authentication. +DEFAULT_AGENT_CONFIG: dict[str, object] = { + "adapter": { + "name": "openai_compatible_react", + "max_iterations": 10, + "provider": { + # Route model calls through the backend by default. The assigned + # profile switches this to a direct provider when it names a + # base_url of its own. + "kind": "code4me_backend", + # base_url / model / api_key_env intentionally omitted — supplied + # by the backend's agent-config after authentication. + }, + "memory_window": { + "scope": "session", + "strategy": "token_window", + "max_messages": 20, + "max_tokens": 32000, + }, + }, + "commands": { + # Programs the agent may not run; any other installed program can run + # (each command still needs the user's approval under per-step + # approval). The server's agent-config replaces this list once one is + # configured. + "denylist": [], + }, + "telemetry": { + "trace_path": ".code4me/acp-trace.jsonl", + # Local JSONL trace detail. Note this does NOT control what the server + # stores: content persistence is decided server-side from the user's + # store_agent_content preference, regardless of this flag. + "raw_capture_enabled": True, + }, +} + +DEFAULT_ACP_JSON: dict[str, object] = { + "default_mcp_settings": { + "use_idea_mcp": True, + "use_custom_mcp": True, + }, + "agent_servers": { + "Code4Me Agent": { + "command": "code4me2-agent", + "args": [], + }, + }, +} + + +def _resolve_agent_command() -> str: + current_command = Path(sys.argv[0]).name + if current_command: + resolved_command = shutil.which(current_command) + if resolved_command: + return resolved_command + + for command in ("code4me2-agent",): + resolved_command = shutil.which(command) + if resolved_command: + return resolved_command + + return sys.argv[0] or "code4me2-agent" + + +def _build_acp_json() -> dict[str, object]: + acp_json = json.loads(json.dumps(DEFAULT_ACP_JSON)) + acp_json["agent_servers"]["Code4Me Agent"]["command"] = _resolve_agent_command() + return acp_json + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run the Code4Me ACP agent over stdio.") + parser.add_argument( + "--config", + default=DEFAULT_CONFIG_PATH, + help="Path to agent config JSON file (default: ~/.code4me/agent-config.json).", + ) + parser.add_argument( + "--setup", + action="store_true", + help="Install default config files to ~/.code4me/ and print setup instructions.", + ) + parser.add_argument( + "--managed", + action="store_true", + help="Run using the authenticated Code4Me IDE bridge and hosted backend.", + ) + parser.add_argument( + "--version", + action="store_true", + help="Print the packaged runtime version and exit.", + ) + parser.add_argument( + "--self-check", + action="store_true", + help="Check runtime dependencies and platform support without changing files.", + ) + return parser + + +def configure_logging() -> None: + level_name = os.environ.get("CODE4ME_AGENT_LOG_LEVEL", "INFO").upper() + level = getattr(logging, level_name, logging.INFO) + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + + +def _do_setup() -> None: + config_dir = Path("~/.code4me").expanduser() + config_dir.mkdir(parents=True, exist_ok=True) + + agent_config_path = config_dir / "agent-config.json" + if not agent_config_path.exists(): + agent_config_path.write_text(json.dumps(DEFAULT_AGENT_CONFIG, indent=2) + "\n") + print(f"Created {agent_config_path}") + else: + print(f"Already exists: {agent_config_path} (skipped)") + + acp_json_path = DEFAULT_ACP_JSON_PATH + if not os.path.exists(acp_json_path): + with open(acp_json_path, "w") as f: + json.dump(_build_acp_json(), f, indent=2) + f.write("\n") + print(f"Created {acp_json_path}") + else: + print(f"Already exists: {acp_json_path} (skipped)") + + print() + print("Setup complete. To use the Code4Me agent in JetBrains AI Assistant:") + print(" 1. Make sure JetBrains AI Assistant points to the acp.json above") + print(" (Settings → Tools → AI Assistant → Custom Agent Server)") + print(" 2. Restart IntelliJ") + print(" 3. Run: docker compose -f docker-compose.dev-arm.yaml up") + print(" 4. Assign an agent profile in the backend (it selects the model/provider;") + print(" the agent has no local model default)") + print(" 5. Run: ./gradlew runIde") + + +def main(argv: list[str] | None = None) -> None: + configure_logging() + args = build_parser().parse_args(argv) + + if args.version: + print(_runtime_version()) + return + + if args.self_check: + raise SystemExit(_self_check()) + + if args.setup: + _do_setup() + sys.exit(0) + + config = _managed_config() if args.managed else AgentConfig.from_file(args.config) + asyncio.run(run_acp_stdio(config)) + + +def _runtime_version() -> str: + return RUNTIME_VERSION + + +def _managed_config() -> AgentConfig: + # Bootstrap only: no tool is available (allowed_tools is empty) until the + # assigned run policy, including its blocked commands, is applied. + workspace = Path.cwd().resolve() + return AgentConfig( + workspace_root=workspace, + trace_path=workspace / ".code4me" / "acp-trace.jsonl", + session_id="managed-bootstrap", + raw_capture_enabled=False, + store_agent_content=False, + commands=CommandConfig(), + adapter=AdapterConfig( + name="openai_compatible_react", + max_iterations=1, + memory_window=MemoryWindowConfig( + scope="session", strategy="token_window", max_messages=20, max_tokens=32000 + ), + provider=OpenAICompatibleProviderConfig(kind="managed_backend"), + ), + allowed_tools=frozenset(), + managed_mode=True, + ) + + +def _self_check() -> int: + checks = { + "runtime_version": _runtime_version(), + "platform": sys.platform, + "platform_supported": sys.platform in SUPPORTED_MANAGED_PLATFORMS, + "python": platform.python_version(), + "frozen": bool(getattr(sys, "frozen", False)), + "os": platform.system(), + "architecture": platform.machine(), + } + try: + import acp # noqa: F401 + import openai # noqa: F401 + from acp.agent.router import build_agent_router # noqa: F401 + from mcp import Client, StdioServerParameters # noqa: F401 + + checks["dependencies"] = "ok" + except ImportError as exc: + checks["dependencies"] = f"missing: {exc.name}" + ok = checks["platform_supported"] and checks["dependencies"] == "ok" + checks["status"] = "ok" if ok else "failed" + print(json.dumps(checks, sort_keys=True)) + return 0 if ok else 1 + + +if __name__ == "__main__": + main() diff --git a/src/code4me2_agent/command_tools.py b/src/code4me2_agent/command_tools.py new file mode 100644 index 0000000..da7a62c --- /dev/null +++ b/src/code4me2_agent/command_tools.py @@ -0,0 +1,1252 @@ +from __future__ import annotations + +import inspect +import os +import re +import shutil +import signal +import subprocess +from dataclasses import dataclass +from pathlib import Path +from threading import Event, Lock, Thread +from time import monotonic, perf_counter +from typing import IO, TYPE_CHECKING, Any, Callable, Iterable, Sequence +from uuid import uuid4 + +from code4me2_agent.acp_utils import capability_value +from code4me2_agent.async_bridge import run_awaitable_blocking +from code4me2_agent.runner_output import summarize_test_output +from code4me2_agent.telemetry import AgentTelemetryRecorder +from code4me2_agent.tool_errors import CommandNotFoundError, ToolError + +if TYPE_CHECKING: + from code4me2_agent.config import AgentConfig + + +_POLL_SECONDS = 0.25 +_KILL_GRACE_SECONDS = 5.0 +_PIPE_CHUNK_BYTES = 65536 +# Streaming of running output into the tool card: at most one update per +# interval, carrying the last lines of output. +_STREAM_INTERVAL_SECONDS = 1.0 +_STREAM_TAIL_LINES = 20 +_STREAM_TAIL_BYTES = 4096 +# Passing test runs keep only their last lines: the summary says what matters. +_PASSING_TEST_OUTPUT_LINES = 20 + +# Build wrappers that live in the project rather than on PATH: a bare +# ``gradlew`` runs the workspace's own copy when none is on PATH. +WORKSPACE_WRAPPERS = frozenset({"gradlew", "gradlew.bat", "mvnw", "mvnw.cmd"}) +# Characters cmd.exe interprets in a batch file's arguments ("BatBadBut"). +_BATCH_UNSAFE_RE = re.compile(r'[&|<>^%"!\r\n]') + +# Blocked-command matching (the study's ``commands_denylist``). Mirrors +# ``blocked_command_in`` in the backend's ``agents/tools.py``, which validates +# profiles with the same rule; keep the two in step. +_LAUNCHER_SUFFIXES = (".exe", ".cmd", ".bat", ".com") +# Shells, launchers and package managers: every word of their arguments is +# checked too, so ``bash -c "git push"``, ``env git push`` or ``npm x git`` cannot +# start a blocked ``git``. A package manager's subcommands and aliases that run +# a program are many (``uv run``, ``npm exec``/``x``/``explore``, ``mise x``), so +# all its arguments count: a package named like a blocked program is refused too. +_COMMAND_RUNNERS = frozenset( + { + "bash", "sh", "zsh", "dash", "ksh", "fish", "csh", "tcsh", "busybox", + "cmd", "powershell", "pwsh", "wsl", + "env", "xargs", "sudo", "doas", "nohup", "nice", "ionice", "setsid", + "stdbuf", "timeout", "gtimeout", "time", "watch", "command", "exec", + "script", "arch", "caffeinate", "chroot", "su", "nix-shell", "npx", "uvx", "bunx", + "uv", "poetry", "pipenv", "pdm", "hatch", "rye", "pixi", "conda", "mamba", + "micromamba", "pipx", "npm", "pnpm", "yarn", "bun", "bundle", "corepack", + "mise", "asdf", "direnv", "pyenv", "rbenv", "nodenv", "nvm", "volta", + } +) +# Quotes only group a word (``g''it`` is ``git``); the rest separate words, +# including ``@`` before a version (``npx npm@10``, ``corepack pnpm@9``). +_QUOTES = re.compile(r"[\'\"]") +_WORD_SEPARATORS = re.compile(r"[\s;&|()<>`{}$=,@]+") + + +def command_key(name: str) -> str: + """The comparable program name: last path component, lower-cased, no ``.exe``. + + Trailing dots and spaces are dropped, as Windows drops them (``git.exe.`` + starts ``git.exe``). + """ + base = re.split(r"[\\/]", name.strip().rstrip("\\/"))[-1].lower().rstrip(" .") + for suffix in _LAUNCHER_SUFFIXES: + if base.endswith(suffix) and len(base) > len(suffix): + return base[: -len(suffix)].rstrip(" .") + return base + + +def blocked_program(argv: Sequence[str], blocked: Iterable[str]) -> str | None: + """The blocked program ``argv`` would start, else ``None``. + + ``argv[0]`` is compared by name (case-insensitive, any directory and + launcher suffix ignored). When it is a shell, a launcher or a package + manager, every plainly written word of its arguments is compared as well + (``npm@10`` counts as ``npm``). Programs started by scripts, interpreters + or build tools, and names disguised with escapes, are not visible here. + """ + keys = {command_key(name) for name in blocked} + keys.discard("") + if not argv or not keys: + return None + program = command_key(argv[0]) + if program in keys: + return program + if program not in _COMMAND_RUNNERS: + return None + for argument in argv[1:]: + for word in _WORD_SEPARATORS.split(_QUOTES.sub("", argument)): + if word and command_key(word) in keys: + return command_key(word) + return None + + +@dataclass(frozen=True) +class CommandResult: + argv: list[str] + cwd: str + backend_type: str + exit_code: int | None + stdout: str + stderr: str + timed_out: bool + output_truncated: bool + duration_ms: float = 0.0 + timeout_seconds: float = 0.0 + status: str = "completed" + # Parsed counts and failing tests when argv ran a recognised test runner. + test_summary: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class AcpCommandExecution: + exit_code: int | None + stdout: str + stderr: str + timed_out: bool + output_truncated: bool + + +class AcpCommandBackend: + def __init__(self, *, client: object, session_id: str, async_runner: object | None = None) -> None: + self._client = client + self._session_id = session_id + self._async_runner = async_runner + + @property + def terminal_enabled(self) -> bool: + return True + + def run_command( + self, + *, + command: str, + args: list[str], + cwd: str, + max_output_bytes: int, + timeout_seconds: float, + ) -> AcpCommandExecution: + terminal_id = self._create_terminal( + command=command, + args=args, + cwd=cwd, + output_byte_limit=max_output_bytes, + ) + + timed_out = False + wait_response: object | None = None + try: + try: + wait_response = self._wait_for_exit(terminal_id=terminal_id, timeout_seconds=timeout_seconds) + except TimeoutError: + timed_out = True + self._kill_terminal(terminal_id=terminal_id) + + output_response = self._terminal_output(terminal_id=terminal_id) + stdout, stderr, output_truncated, output_exit_code = self._parse_output_response(output_response) + exit_code = self._parse_exit_code(wait_response) + if exit_code is None: + exit_code = output_exit_code + return AcpCommandExecution( + exit_code=exit_code, + stdout=stdout, + stderr=stderr, + timed_out=timed_out, + output_truncated=output_truncated, + ) + finally: + self._release_terminal(terminal_id=terminal_id) + + def _create_terminal( + self, + *, + command: str, + args: list[str], + cwd: str, + output_byte_limit: int, + ) -> str: + method = _resolve_method(self._client, ["create_terminal", "terminal_create"]) + response = self._run_client_call( + _invoke_with_name_fallback( + method, + snake_kwargs={ + "session_id": self._session_id, + "command": command, + "args": args, + "cwd": cwd, + "output_byte_limit": output_byte_limit, + }, + camel_kwargs={ + "sessionId": self._session_id, + "command": command, + "args": args, + "cwd": cwd, + "outputByteLimit": output_byte_limit, + }, + ) + ) + if isinstance(response, dict): + terminal_id = response.get("terminal_id") or response.get("terminalId") + else: + terminal_id = getattr(response, "terminal_id", None) or getattr(response, "terminalId", None) + if terminal_id is None: + raise RuntimeError("ACP terminal backend did not return a terminal ID.") + return str(terminal_id) + + def _wait_for_exit(self, *, terminal_id: str, timeout_seconds: float) -> object: + method = _resolve_method( + self._client, + ["wait_for_terminal_exit", "terminal_wait_for_exit"], + ) + return self._run_client_call( + _invoke_with_name_fallback( + method, + snake_kwargs={ + "session_id": self._session_id, + "terminal_id": terminal_id, + }, + camel_kwargs={ + "sessionId": self._session_id, + "terminalId": terminal_id, + }, + ), + timeout_seconds=timeout_seconds, + ) + + def _terminal_output(self, *, terminal_id: str) -> object: + method = _resolve_method( + self._client, + ["get_terminal_output", "terminal_output"], + ) + return self._run_client_call( + _invoke_with_name_fallback( + method, + snake_kwargs={ + "session_id": self._session_id, + "terminal_id": terminal_id, + }, + camel_kwargs={ + "sessionId": self._session_id, + "terminalId": terminal_id, + }, + ) + ) + + def _kill_terminal(self, *, terminal_id: str) -> None: + method = _resolve_method(self._client, ["kill_terminal", "terminal_kill"]) + self._run_client_call( + _invoke_with_name_fallback( + method, + snake_kwargs={ + "session_id": self._session_id, + "terminal_id": terminal_id, + }, + camel_kwargs={ + "sessionId": self._session_id, + "terminalId": terminal_id, + }, + ) + ) + + def _release_terminal(self, *, terminal_id: str) -> None: + method = _resolve_method(self._client, ["release_terminal", "terminal_release"]) + self._run_client_call( + _invoke_with_name_fallback( + method, + snake_kwargs={ + "session_id": self._session_id, + "terminal_id": terminal_id, + }, + camel_kwargs={ + "sessionId": self._session_id, + "terminalId": terminal_id, + }, + ) + ) + + def _run_client_call(self, result: object, timeout_seconds: float | None = None) -> object: + if not inspect.isawaitable(result): + return result + if self._async_runner is not None: + return self._async_runner.run(result, timeout_seconds=timeout_seconds) + return run_awaitable_blocking(result, timeout_seconds=timeout_seconds) + + def _parse_output_response(self, response: object) -> tuple[str, str, bool, int | None]: + if isinstance(response, dict): + output = str(response.get("output", "")) + stderr = str(response.get("stderr", "")) + truncated = bool(response.get("truncated", False)) + exit_status = response.get("exit_status") or response.get("exitStatus") or {} + else: + output = str(getattr(response, "output", "")) + stderr = str(getattr(response, "stderr", "")) + truncated = bool(getattr(response, "truncated", False)) + exit_status = getattr(response, "exit_status", None) or getattr(response, "exitStatus", None) or {} + exit_code = _extract_exit_code(exit_status) + return output, stderr, truncated, exit_code + + def _parse_exit_code(self, response: object | None) -> int | None: + if response is None: + return None + return _extract_exit_code(response) + + +def build_acp_command_backend( + *, + client: object, + session_id: str, + client_capabilities: object | None, + async_runner: object | None = None, +) -> AcpCommandBackend | None: + terminal_capability = capability_value(client_capabilities, "terminal") + if not bool(terminal_capability): + return None + + required_method_groups = [ + ["create_terminal", "terminal_create"], + ["wait_for_terminal_exit", "terminal_wait_for_exit"], + ["get_terminal_output", "terminal_output"], + ["release_terminal", "terminal_release"], + ["kill_terminal", "terminal_kill"], + ] + for names in required_method_groups: + if _resolve_method(client, names) is None: + return None + + return AcpCommandBackend(client=client, session_id=session_id, async_runner=async_runner) + + +class WorkspaceCommandTools: + def __init__( + self, + config: AgentConfig, + *, + acp_backend: object | None = None, + telemetry: AgentTelemetryRecorder | None = None, + blocked_commands: Iterable[str] | None = None, + timeout_seconds: float | None = None, + max_timeout_seconds: float | None = None, + max_output_bytes: int | None = None, + ) -> None: + self._config = config + self._acp_backend = acp_backend + self._telemetry = telemetry or AgentTelemetryRecorder(config) + command_config = config.commands + self._blocked_commands = tuple( + command_config.blocked_commands if blocked_commands is None else blocked_commands + ) + selected_timeout_seconds = command_config.timeout_seconds if timeout_seconds is None else timeout_seconds + selected_max_timeout_seconds = ( + getattr(command_config, "max_timeout_seconds", 600.0) + if max_timeout_seconds is None + else max_timeout_seconds + ) + selected_max_output_bytes = command_config.max_output_bytes if max_output_bytes is None else max_output_bytes + self._timeout_seconds = max(1.0, float(selected_timeout_seconds)) + self._max_timeout_seconds = max(self._timeout_seconds, float(selected_max_timeout_seconds)) + self._max_output_bytes = max(1, int(selected_max_output_bytes)) + + @property + def blocked_commands(self) -> tuple[str, ...]: + """The programs this session may not run (read-only; tool descriptions follow it).""" + return self._blocked_commands + + @property + def workspace_root(self) -> Path: + return self._config.workspace_root + + def run_command( + self, + argv: object, + *, + cwd: str = ".", + timeout_seconds: float | None = None, + cancel_event: Event | None = None, + on_output: Callable[[str], None] | None = None, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> CommandResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + + normalized_argv = self._normalize_argv_or_record_denial( + argv=argv, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + command_name = _program_name(normalized_argv[0]) + self._check_blocked_or_record_denial( + argv=normalized_argv, + cwd=cwd, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + resolved_cwd = self._resolve_cwd_or_record_denial( + cwd=cwd, + argv=normalized_argv, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + executable = self._executable_path( + command_name=command_name, argv=normalized_argv, cwd=resolved_cwd + ) + self._check_batch_arguments_or_record_denial( + program=executable or shutil.which(command_name) or command_name, + command_name=command_name, + argv=normalized_argv, + cwd=resolved_cwd, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + effective_timeout = self._effective_timeout(timeout_seconds) + + relative_cwd = "." if resolved_cwd == self._config.workspace_root else self._relative_path(resolved_cwd) + acp_run_command = self._acp_run_command() + cancelled = False + if acp_run_command is not None: + backend_type = "acp" + acp_result = acp_run_command( + command=executable or command_name, + args=normalized_argv[1:], + cwd=str(resolved_cwd), + max_output_bytes=self._max_output_bytes, + timeout_seconds=effective_timeout, + ) + stdout, stdout_truncated = _truncate_to_max_bytes( + clean_terminal_output(acp_result.stdout), self._max_output_bytes + ) + stderr, stderr_truncated = _truncate_to_max_bytes( + clean_terminal_output(acp_result.stderr), self._max_output_bytes + ) + exit_code = acp_result.exit_code + timed_out = acp_result.timed_out + output_truncated = acp_result.output_truncated or stdout_truncated or stderr_truncated + else: + backend_type = "local" + local = self._run_local( + argv=[executable, *normalized_argv[1:]] if executable else normalized_argv, + cwd=resolved_cwd, + timeout_seconds=effective_timeout, + cancel_event=cancel_event, + on_output=on_output, + ) + stdout, stderr = local.stdout, local.stderr + exit_code = local.exit_code + timed_out = local.timed_out + cancelled = local.cancelled + output_truncated = local.output_truncated + + if cancelled: + status = "cancelled" + elif timed_out: + status = "timeout" + else: + status = "completed" + test_summary = None + if status == "completed" and self._test_summaries_enabled(): + summary = summarize_test_output(normalized_argv, stdout, stderr) + if summary is not None: + test_summary = summary.as_result() + if exit_code == 0 and summary.all_passed: + stdout = _last_lines(stdout, _PASSING_TEST_OUTPUT_LINES) + duration_ms = round((perf_counter() - started_at) * 1000, 3) + + self._record_tool_event( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + argv=normalized_argv, + cwd=relative_cwd, + status=status, + backend_type=backend_type, + started_at=started_at, + exit_code=exit_code, + stdout=stdout, + stderr=stderr, + timed_out=timed_out, + output_truncated=output_truncated, + extra_payload={ + "command": command_name, + "stdout_bytes": len(stdout.encode("utf-8")), + "stderr_bytes": len(stderr.encode("utf-8")), + "max_output_bytes": self._max_output_bytes, + "timeout_seconds": effective_timeout, + **_test_summary_payload(test_summary), + }, + ) + + return CommandResult( + argv=normalized_argv, + cwd=relative_cwd, + backend_type=backend_type, + exit_code=exit_code, + stdout=stdout, + stderr=stderr, + timed_out=timed_out, + output_truncated=output_truncated, + duration_ms=duration_ms, + timeout_seconds=effective_timeout, + status=status, + test_summary=test_summary, + ) + + def _test_summaries_enabled(self) -> bool: + harness = getattr(self._config, "harness", None) + return bool(getattr(harness, "test_output_summary", True)) + + def _effective_timeout(self, requested: float | None) -> float: + if requested is None: + value = self._timeout_seconds + else: + try: + value = float(requested) + except (TypeError, ValueError): + value = self._timeout_seconds + return min(max(value, 1.0), self._max_timeout_seconds) + + def _run_local( + self, + *, + argv: list[str], + cwd: Path, + timeout_seconds: float, + cancel_event: Event | None, + on_output: Callable[[str], None] | None = None, + ) -> "_LocalExecution": + popen_kwargs: dict[str, Any] = { + "cwd": str(cwd), + "stdin": subprocess.DEVNULL, + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "env": command_environment(), + "close_fds": True, + } + if os.name == "nt": + popen_kwargs["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) + else: + popen_kwargs["start_new_session"] = True + try: + process = subprocess.Popen(argv, **popen_kwargs) + except FileNotFoundError: + raise CommandNotFoundError( + f"Command not found: {argv[0]}. Run a program that is installed on this " + "machine, or give the path of a script in the project." + ) from None + except PermissionError as exc: + raise ToolError( + f"Command is not executable: {argv[0]} ({exc.strerror or exc}).", + code="command_not_executable", + ) from None + + stdout_collector = _PipeCollector(process.stdout, self._max_output_bytes) + stderr_collector = _PipeCollector(process.stderr, self._max_output_bytes) + stdout_collector.start() + stderr_collector.start() + + deadline = monotonic() + timeout_seconds + timed_out = False + cancelled = False + exit_code: int | None = None + next_stream = monotonic() + _STREAM_INTERVAL_SECONDS + streamed_total = -1 + while True: + try: + exit_code = process.wait(timeout=_POLL_SECONDS) + break + except subprocess.TimeoutExpired: + if on_output is not None and monotonic() >= next_stream: + next_stream = monotonic() + _STREAM_INTERVAL_SECONDS + total = stdout_collector.total + stderr_collector.total + if total != streamed_total: + streamed_total = total + _deliver_stream(on_output, stdout_collector, stderr_collector) + if cancel_event is not None and cancel_event.is_set(): + cancelled = True + elif monotonic() >= deadline: + timed_out = True + else: + continue + _terminate_process_tree(process) + try: + process.wait(timeout=_KILL_GRACE_SECONDS) + except subprocess.TimeoutExpired: + pass + exit_code = None + break + stdout_collector.join(timeout=_KILL_GRACE_SECONDS) + stderr_collector.join(timeout=_KILL_GRACE_SECONDS) + stdout_text, stdout_truncated = stdout_collector.render() + stderr_text, stderr_truncated = stderr_collector.render() + return _LocalExecution( + stdout=clean_terminal_output(stdout_text), + stderr=clean_terminal_output(stderr_text), + exit_code=exit_code, + timed_out=timed_out, + cancelled=cancelled, + output_truncated=stdout_truncated or stderr_truncated, + ) + + def _acp_run_command(self) -> Any | None: + if not bool(getattr(self._acp_backend, "terminal_enabled", False)): + return None + operation = getattr(self._acp_backend, "run_command", None) + if callable(operation): + return operation + return None + + def _executable_path(self, *, command_name: str, argv: list[str], cwd: Path) -> str | None: + """Absolute path of the program when argv[0] names a file, else None (PATH lookup). + + A path (``./gradlew``, ``scripts/test.sh``, ``/usr/bin/git``) is resolved + against the working directory without following symlinks (a virtualenv's + ``python`` must keep its own path). A bare project build wrapper + (``gradlew``) that is not on PATH is looked up in the working directory + and then the workspace root. + """ + program = argv[0] + if "/" in program or "\\" in program: + candidate = Path(program) + if not candidate.is_absolute(): + candidate = cwd / candidate + if candidate.is_file(): + return os.path.abspath(candidate) + # Not a policy decision: the adapter records this as a failed call. + raise CommandNotFoundError( + f"{program} was not found; a relative path is resolved against cwd " + f"({self._relative_path(cwd)})." + ) + if command_name not in WORKSPACE_WRAPPERS or shutil.which(command_name): + return None + for candidate in (cwd / command_name, self._config.workspace_root / command_name): + if candidate.is_file(): + return os.path.abspath(candidate) + raise CommandNotFoundError( + f"{program} was not found in the workspace. Build wrappers run from the project, " + f'e.g. ["./{command_name}", ...] with cwd set to the directory that contains it.' + ) + + def _check_batch_arguments_or_record_denial( + self, + *, + program: str, + command_name: str, + argv: list[str], + cwd: Path, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> None: + """Refuse cmd.exe metacharacters in the arguments of a batch file. + + Windows runs ``.bat``/``.cmd`` files through cmd.exe, which re-parses + the command line ("BatBadBut"): ``&``, ``|``, ``%`` and quotes in an + argument would run other commands, unseen by the blocked-command check. + This applies to a project wrapper and to a batch file found on PATH alike. + """ + # Windows drops trailing dots and spaces: ``npm.cmd.`` is ``npm.cmd``. + is_batch = any( + Path(name.rstrip(" .")).suffix.lower() in (".bat", ".cmd") + for name in (program, command_name) + ) + if not is_batch or not any(_BATCH_UNSAFE_RE.search(argument) for argument in argv[1:]): + return + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=argv, + cwd=self._relative_path(cwd), + denial_reason="unsafe_batch_arguments", + ) + raise PermissionError( + f"{command_name} runs as a batch file: its arguments must not contain " + '& | < > ^ % " ! or line breaks.' + ) + + def _normalize_argv_or_record_denial( + self, + *, + argv: object, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> list[str]: + if isinstance(argv, str): + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=[argv], + cwd=".", + denial_reason="argv_must_be_array", + ) + raise TypeError("Command execution requires argv as a list/tuple, not a shell string.") + + if not isinstance(argv, (list, tuple)): + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=[], + cwd=".", + denial_reason="invalid_argv", + ) + raise TypeError("argv must be a list or tuple of strings.") + + normalized: list[str] = [] + for arg in argv: + if not isinstance(arg, str): + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=[str(item) for item in argv], + cwd=".", + denial_reason="invalid_argv", + ) + raise TypeError("argv must contain only strings.") + normalized.append(arg) + + if not normalized or not normalized[0].strip(): + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=normalized, + cwd=".", + denial_reason="invalid_argv", + ) + raise ValueError("argv must contain a non-empty command at index 0.") + + return normalized + + def _check_blocked_or_record_denial( + self, + *, + argv: list[str], + cwd: str, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> None: + blocked = blocked_program(argv, self._blocked_commands) + if blocked is None: + return + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=argv, + cwd=cwd, + denial_reason="command_blocked", + ) + via = "" if command_key(argv[0]) == blocked else f" (through {_program_name(argv[0])})" + raise PermissionError( + f"{blocked} is blocked in this study and cannot run{via}. Blocked programs: " + f"{', '.join(self._blocked_commands)}. Do not try to run it another way; " + "use a different approach or tell the user." + ) + + def _resolve_cwd_or_record_denial( + self, + *, + cwd: str, + argv: list[str], + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> Path: + try: + return self._resolve_workspace_path(cwd) + except PermissionError: + self._record_denial( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + argv=argv, + cwd=cwd, + denial_reason="outside_workspace_root", + ) + raise + + def _resolve_workspace_path(self, path: str) -> Path: + workspace_root = self._config.workspace_root.resolve() + requested_path = Path(str(path)).expanduser() + if not requested_path.is_absolute(): + requested_path = workspace_root / requested_path + resolved_path = requested_path.resolve() + if resolved_path != workspace_root and workspace_root not in resolved_path.parents: + raise PermissionError("Path is outside the configured workspace root.") + return resolved_path + + def _relative_path(self, path: Path) -> str: + try: + return path.relative_to(self._config.workspace_root).as_posix() + except ValueError: + return path.as_posix() + + def _record_denial( + self, + *, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + argv: list[str], + cwd: str, + denial_reason: str, + ) -> None: + self._record_tool_event( + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + argv=argv, + cwd=cwd, + status="denied", + backend_type="local", + started_at=started_at, + exit_code=None, + stdout="", + stderr="", + timed_out=False, + output_truncated=False, + denial_reason=denial_reason, + ) + + def _record_tool_event( + self, + *, + tool_call_id: str, + run_id: str, + request_id: str, + argv: list[str], + cwd: str, + status: str, + backend_type: str, + started_at: float, + exit_code: int | None, + stdout: str, + stderr: str, + timed_out: bool, + output_truncated: bool, + denial_reason: str | None = None, + extra_payload: dict[str, object] | None = None, + ) -> None: + event_type = "agent.tool.denied" if status == "denied" else "agent.tool.completed" + payload: dict[str, object] = { + "tool_name": "run_command", + "tool_call_id": tool_call_id, + "backend_type": backend_type, + "duration_ms": round((perf_counter() - started_at) * 1000, 3), + "status": status, + "argv": argv, + "cwd": cwd, + "exit_code": exit_code, + "timed_out": timed_out, + "output_truncated": output_truncated, + "stdout": stdout, + "stderr": stderr, + } + payload.update(extra_payload or {}) + if denial_reason is not None: + payload["denial_reason"] = denial_reason + + self._telemetry.record( + event_type=event_type, + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload=payload, + raw_payload=None, + ) + + +@dataclass(frozen=True) +class _LocalExecution: + stdout: str + stderr: str + exit_code: int | None + timed_out: bool + cancelled: bool + output_truncated: bool + + +class _PipeCollector(Thread): + """Drain a pipe keeping the first and last bytes within a fixed memory budget.""" + + def __init__(self, pipe: IO[bytes] | None, max_bytes: int) -> None: + super().__init__(daemon=True) + self._pipe = pipe + self._max_bytes = max(1, int(max_bytes)) + self._head_limit = max(1, self._max_bytes // 3) + self._tail_limit = max(1, self._max_bytes - self._head_limit) + self._head = bytearray() + self._tail = bytearray() + self._total = 0 + self._lock = Lock() + + @property + def total(self) -> int: + return self._total + + def recent(self, max_bytes: int) -> bytes: + """The most recent output so far (for streaming into the tool card).""" + with self._lock: + if self._tail: + return bytes(self._tail[-max_bytes:]) + return bytes(self._head[-max_bytes:]) + + def run(self) -> None: + if self._pipe is None: + return + try: + while True: + chunk = self._pipe.read1(_PIPE_CHUNK_BYTES) if hasattr(self._pipe, "read1") else self._pipe.read(_PIPE_CHUNK_BYTES) + if not chunk: + break + with self._lock: + self._total += len(chunk) + if len(self._head) < self._head_limit: + take = self._head_limit - len(self._head) + self._head += chunk[:take] + chunk = chunk[take:] + if chunk: + self._tail += chunk + if len(self._tail) > 2 * self._tail_limit: + del self._tail[: len(self._tail) - self._tail_limit] + except (OSError, ValueError): + pass + finally: + try: + self._pipe.close() + except OSError: + pass + + def render(self) -> tuple[str, bool]: + with self._lock: + return self._render_locked() + + def _render_locked(self) -> tuple[str, bool]: + tail = bytes(self._tail[-self._tail_limit :]) if self._tail else b"" + if self._total <= self._max_bytes: + return (bytes(self._head) + tail).decode("utf-8", errors="replace"), False + dropped = self._total - len(self._head) - len(tail) + text = ( + bytes(self._head).decode("utf-8", errors="replace") + + f"\n[... {dropped} bytes truncated ...]\n" + + tail.decode("utf-8", errors="replace") + ) + return text, True + + +def _terminate_process_tree(process: subprocess.Popen[bytes]) -> None: + if os.name == "nt": + try: + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(process.pid)], + capture_output=True, + timeout=10, + check=False, + ) + except Exception: # noqa: BLE001 + pass + try: + process.kill() + except Exception: # noqa: BLE001 + pass + return + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + except (PermissionError, OSError): + try: + process.kill() + except Exception: # noqa: BLE001 + pass + + +def _program_name(program: str) -> str: + """The program a command starts: argv[0] without its directory.""" + return re.split(r"[\\/]", program.rstrip("\\/"))[-1] or program + + +def available_commands(commands: list[str]) -> list[str]: + """Return the commands that can actually launch on this machine. + + Project build wrappers (``gradlew``, ``mvnw``) are kept: they live in the + workspace, not on PATH, and are resolved per call. + """ + return [ + command + for command in commands + if command in WORKSPACE_WRAPPERS or shutil.which(command) + ] + + +def _deliver_stream( + on_output: Callable[[str], None], + stdout_collector: "_PipeCollector", + stderr_collector: "_PipeCollector", +) -> None: + parts = [] + for collector in (stdout_collector, stderr_collector): + recent = collector.recent(_STREAM_TAIL_BYTES) + if recent: + parts.append(clean_terminal_output(recent.decode("utf-8", errors="replace"))) + text = _last_lines("\n".join(part.rstrip("\n") for part in parts), _STREAM_TAIL_LINES) + if not text.strip(): + return + try: + on_output(text) + except Exception: # noqa: BLE001 - a card update must never kill the command + pass + + +def _last_lines(text: str, count: int) -> str: + lines = text.rstrip("\n").splitlines() + if len(lines) <= count: + return text + return f"[... {len(lines) - count} earlier lines omitted ...]\n" + "\n".join(lines[-count:]) + + +def _test_summary_payload(test_summary: dict[str, Any] | None) -> dict[str, Any]: + if not test_summary: + return {} + payload: dict[str, Any] = {"test_framework": test_summary.get("framework")} + for key in ("passed", "failed", "errors", "skipped"): + if test_summary.get(key) is not None: + payload[f"tests_{key}"] = test_summary[key] + return payload + + +# ANSI CSI/OSC sequences, other escapes and non-printing control characters. +_ANSI_ESCAPE_RE = re.compile( + r"\x1b\[[0-?]*[ -/]*[@-~]" # CSI: colours, cursor moves, erase line + r"|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)" # OSC: titles, hyperlinks + r"|\x1b[@-Z\\-_]" # two-character escapes +) +_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") + + +def clean_terminal_output(text: str) -> str: + """Strip colour codes and collapse carriage-return progress redraws.""" + if not text: + return text + if "\x1b" in text: + text = _ANSI_ESCAPE_RE.sub("", text) + if "\r" in text: + text = text.replace("\r\n", "\n") + lines = [] + for line in text.split("\n"): + if "\r" in line: + segments = [segment for segment in line.split("\r") if segment] + line = segments[-1] if segments else "" + lines.append(line) + text = "\n".join(lines) + return _CONTROL_RE.sub("", text) + + +_ENV_EXACT = frozenset( + { + "PATH", + "HOME", + "USER", + "LOGNAME", + "SHELL", + "TERM", + "LANG", + "LANGUAGE", + "TZ", + "TMPDIR", + "TEMP", + "TMP", + "SYSTEMROOT", + "SYSTEMDRIVE", + "WINDIR", + "COMSPEC", + "PATHEXT", + "USERPROFILE", + "APPDATA", + "LOCALAPPDATA", + "PROGRAMDATA", + "PROGRAMFILES", + "PROGRAMFILES(X86)", + "HOMEDRIVE", + "HOMEPATH", + "USERNAME", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_RUNTIME_DIR", + "JAVA_HOME", + "JDK_HOME", + "KOTLIN_HOME", + "ANDROID_HOME", + "ANDROID_SDK_ROOT", + "M2_HOME", + "MAVEN_HOME", + "MAVEN_OPTS", + "GOPATH", + "GOROOT", + "NVM_DIR", + "VIRTUAL_ENV", + "CONDA_PREFIX", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "SSH_AUTH_SOCK", + "SSH_AGENT_PID", + } +) +_ENV_PREFIXES = ( + "LC_", + "GIT_", + "GRADLE_", + "JAVA_", + "PYTHON", + "NODE_", + "NPM_CONFIG_", + "SBT_", + "DOTNET_", + "CARGO_", + "RUSTUP_", +) +_ENV_BLOCKED_EXACT = frozenset({"PYTHONHOME", "PYTHONEXECUTABLE", "LD_LIBRARY_PATH"}) +_ENV_BLOCKED_PREFIXES = ("CODE4ME_", "_MEIPASS", "DYLD_", "PYINSTALLER") + + +# Non-interactive, colour-free output for captured commands: pagers would wait +# on the closed stdin, colours and progress bars waste tokens. +OUTPUT_HYGIENE_ENVIRONMENT: dict[str, str] = { + "NO_COLOR": "1", + "FORCE_COLOR": "0", + "TERM": "dumb", + "CI": "1", + "PAGER": "cat", + "GIT_PAGER": "cat", + "PYTHONUNBUFFERED": "1", + "GIT_TERMINAL_PROMPT": "0", +} + + +def command_environment(source: dict[str, str] | None = None) -> dict[str, str]: + """The child environment: the allowlisted variables plus output hygiene.""" + environment = _external_command_environment(source) + environment.update(OUTPUT_HYGIENE_ENVIRONMENT) + return environment + + +def _external_command_environment(source: dict[str, str] | None = None) -> dict[str, str]: + """Build the child environment: an allowlist of shell/toolchain variables. + + Credentials the runtime itself needs (``CODE4ME_ACP_TOKEN`` and friends) and + PyInstaller loader variables never reach participant tools; a PyInstaller + ``LD_LIBRARY_PATH_ORIG`` is restored as the child's ``LD_LIBRARY_PATH``. + """ + environment = dict(os.environ if source is None else source) + original_library_path = environment.get("LD_LIBRARY_PATH_ORIG") + child_environment: dict[str, str] = {} + for key, value in environment.items(): + upper = key.upper() + if upper in _ENV_BLOCKED_EXACT or upper.startswith(_ENV_BLOCKED_PREFIXES): + continue + if upper in _ENV_EXACT or upper.startswith(_ENV_PREFIXES): + child_environment[key] = value + if original_library_path: + child_environment["LD_LIBRARY_PATH"] = original_library_path + return child_environment + + +def _resolve_method(client: object, names: list[str]) -> Any | None: + for name in names: + method = getattr(client, name, None) + if callable(method): + return method + return None + + +def _invoke_with_name_fallback(method: Any, *, snake_kwargs: dict[str, object], camel_kwargs: dict[str, object]) -> object: + try: + return method(**snake_kwargs) + except TypeError: + return method(**camel_kwargs) + + +def _extract_exit_code(response: object) -> int | None: + if isinstance(response, dict): + raw_exit_code = response.get("exit_code") + if raw_exit_code is None: + raw_exit_code = response.get("exitCode") + else: + raw_exit_code = getattr(response, "exit_code", None) + if raw_exit_code is None: + raw_exit_code = getattr(response, "exitCode", None) + if raw_exit_code is None: + return None + try: + return int(raw_exit_code) + except (TypeError, ValueError): + return None + + +def _truncate_to_max_bytes(value: str, max_output_bytes: int) -> tuple[str, bool]: + raw_bytes = value.encode("utf-8") + if len(raw_bytes) <= max_output_bytes: + return value, False + + truncated_bytes = raw_bytes[-max_output_bytes:] + while truncated_bytes: + try: + return truncated_bytes.decode("utf-8"), True + except UnicodeDecodeError: + truncated_bytes = truncated_bytes[1:] + return "", True diff --git a/src/code4me2_agent/config.py b/src/code4me2_agent/config.py new file mode 100644 index 0000000..63e859b --- /dev/null +++ b/src/code4me2_agent/config.py @@ -0,0 +1,768 @@ +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable +from uuid import uuid4 + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class UploadConfig: + enabled: bool = False + ingest_url: str | None = None + batch_size: int = 50 + timeout_seconds: float = 5.0 + auth_headers: dict[str, str] = field(default_factory=dict) + auth_headers_provider: Callable[[], dict[str, str]] | None = field( + default=None, repr=False, compare=False + ) + + +@dataclass(frozen=True) +class CommandConfig: + # Programs run_command refuses (the study's ``commands_denylist``); every + # other installed program may run. Empty = nothing blocked. + blocked_commands: list[str] = field(default_factory=list) + # Default per-command timeout; the model may raise it per call up to + # ``max_timeout_seconds`` (builds and test suites routinely exceed 10 s). + timeout_seconds: float = 120.0 + max_timeout_seconds: float = 600.0 + max_output_bytes: int = 16384 + + +@dataclass(frozen=True) +class MemoryWindowConfig: + scope: str = "prompt" + strategy: str = "last_messages" + max_messages: int = 12 + # Estimated tokens (chars/4) of conversation kept per request; the server + # profile's max_context_tokens overrides this. + max_tokens: int = 32000 + + +@dataclass(frozen=True) +class FakeProviderConfig: + enabled: bool = False + script: list[dict[str, object]] = field(default_factory=list) + + +PROMPT_PROFILES = frozenset({"auto", "default", "openai", "anthropic", "gemini"}) + +_HARNESS_BOOL_OPTIONS = ( + "self_review", + "verify_on_stop", + "context_summarization", + "parallel_tools", + "project_instructions", + "read_before_edit", + "syntax_check", + "loop_guard", + "instruction_reminders", + "test_output_summary", + "approval_revise", +) + + +@dataclass(frozen=True) +class HarnessOptions: + """Runtime behaviour switches an agent profile can freeze (``harness_options``). + + Every default is the runtime's recommended behaviour; a profile only lists + the switches it changes. The server validates the keys strictly; the + runtime ignores keys it does not know so an older runtime never refuses a + newer policy for an option it cannot honour anyway. + """ + + self_review: bool = True + verify_on_stop: bool = True + # argv run by the runtime before the final answer when files changed; + # None means "nudge the model to verify" instead. + verify_command: tuple[str, ...] | None = None + context_summarization: bool = True + parallel_tools: bool = True + prompt_profile: str = "auto" + project_instructions: bool = True + read_before_edit: bool = True + syntax_check: bool = True + loop_guard: bool = True + instruction_reminders: bool = True + test_output_summary: bool = True + # Under per_step approval, offer "Revise…" next to allow/reject when the + # client renders ACP forms (elicitation.form). + approval_revise: bool = True + + def with_overrides(self, overrides: dict[str, object] | None) -> "HarnessOptions": + if not overrides: + return self + from dataclasses import replace as _replace + + return _replace(self, **overrides) + + +def parse_harness_overrides(value: object, *, strict: bool) -> dict[str, object] | None: + """Validate a ``harness_options`` mapping into ``HarnessOptions`` overrides. + + ``strict`` (the managed policy) raises ``ValueError`` on a known key with a + wrong type; otherwise such a key is skipped. Unknown keys are ignored. + """ + if value is None: + return None + if not isinstance(value, dict): + if strict: + raise ValueError("harness_options must be an object.") + return None + overrides: dict[str, object] = {} + for key in _HARNESS_BOOL_OPTIONS: + if key not in value: + continue + if isinstance(value[key], bool): + overrides[key] = value[key] + elif strict: + raise ValueError(f"harness_options.{key} must be a boolean.") + if "prompt_profile" in value: + profile = value["prompt_profile"] + if isinstance(profile, str) and profile.strip().lower() in PROMPT_PROFILES: + overrides["prompt_profile"] = profile.strip().lower() + elif strict: + raise ValueError("harness_options.prompt_profile is not a known profile.") + if "verify_command" in value: + command = value["verify_command"] + if command is None: + overrides["verify_command"] = None + elif ( + isinstance(command, list) + and command + and all(isinstance(item, str) and item for item in command) + and command[0].strip() + ): + # Same rule as the server (agents/tools.py): non-empty arguments; a + # whitespace argument such as " " is a legitimate argv item. + overrides["verify_command"] = tuple(str(item) for item in command) + elif strict: + raise ValueError("harness_options.verify_command must be a non-empty argv list.") + return overrides + + +@dataclass(frozen=True) +class OpenAICompatibleProviderConfig: + """Where this agent sends its model calls. + + There is deliberately **no hardcoded model or provider default here.** The + backend is authoritative: after ACP authentication the runtime fetches + ``GET /api/acp/agent-config`` and overwrites these fields from the agent + profile the user is assigned to (see ``AgentConfig.with_server_overrides``). + Baking in a local Ollama default — as the original did — meant a study + assignment could be silently ignored whenever the local config disagreed, + which is exactly the failure the A/B design can't tolerate. + + ``kind`` selects how the call is routed: + + * ``code4me_backend`` — through the backend's own OpenAI-compatible bridge. + Used when the assigned profile leaves ``base_url`` unset, so the backend + picks the upstream and the call is observable server-side too. + * ``openai`` — straight to the ``base_url`` the profile names. Any + OpenAI-compatible endpoint qualifies (Ollama, Groq, OpenRouter, OpenAI); + per merge decision 6 the wire format is the only requirement. + + ``api_key_env`` is the *name* of the environment variable holding the + credential, never the credential itself: the backend sends only the name + (``api_key_ref``) and the key is resolved here, in the agent's own process. + """ + + kind: str = "code4me_backend" + # Empty = not yet configured. Filled from the server's agent-config; for + # `code4me_backend` this is the backend's own root URL. + base_url: str = "" + model: str = "" + api_key_env: str = "" + timeout_seconds: float = 90.0 + temperature: float | None = None + auth_headers: dict[str, str] = field(default_factory=dict) + # Forwarded as ``max_tokens`` only when set: newer OpenAI models reject it in + # favour of ``max_completion_tokens`` and the relay hides which upstream is used. + max_output_tokens: int | None = None + # Wall-clock limit for one direct model request; None = only the SDK's read + # timeout. DeepSeek keeps queued non-streaming requests alive with blank + # lines for up to 30 minutes, which a read timeout never notices. + request_deadline_seconds: float | None = None + # Extra top-level fields for every direct (``openai``) request body, such as + # OpenRouter's ``provider`` routing that pins one upstream. Local config + # only: a server-assigned provider never carries them. + extra_body: dict[str, Any] = field(default_factory=dict) + + @property + def is_configured(self) -> bool: + """Whether this provider has enough to make a call.""" + return bool(self.base_url.strip() and self.model.strip()) + + +@dataclass(frozen=True) +class AdapterConfig: + name: str = "deterministic_echo" + max_iterations: int = 8 + memory_window: MemoryWindowConfig = field(default_factory=MemoryWindowConfig) + fake_provider: FakeProviderConfig = field(default_factory=FakeProviderConfig) + provider: OpenAICompatibleProviderConfig = field( + default_factory=OpenAICompatibleProviderConfig + ) + # Researcher-authored system prompt override (ported from origin/sys_prompt). + # None = the runtime's built-in prompt; when set it replaces the persona + # paragraph, while the operational tool/policy instructions are kept. + system_prompt: str | None = None + + +@dataclass(frozen=True) +class ServerAgentConfig: + """The runtime configuration the backend hands down. + + Mirrors the ``GET /api/acp/agent-config`` response, which derives every + field from the agent profile the user is assigned to. All fields are + optional so a partial response (or an older backend) leaves the + corresponding local value untouched rather than blanking it. + """ + + agent_profile: str | None = None + framework_version: str | None = None + model: str | None = None + base_url: str | None = None + api_key_ref: str | None = None + # Programs the agent may not run. The server also sends the legacy + # ``commands_allowlist`` for runtimes 0.0.6 and earlier; this runtime + # ignores it. + commands_denylist: list[str] | None = None + tools: list[str] | None = None + max_iterations: int | None = None + max_context_tokens: int | None = None + approval_policy: str | None = None + temperature: float | None = None + # Researcher-authored system prompt for the assigned arm (agent-config and + # the managed run policy both carry it). None = built-in default. + system_prompt: str | None = None + # Default per-command timeout frozen with the profile. None = runtime default. + command_timeout_seconds: int | None = None + # Validated ``HarnessOptions`` overrides frozen with the profile. + harness_options: dict[str, object] | None = None + # Advisory: the server enforces content storage itself. Used only to avoid + # transmitting content that would be discarded anyway — never to enable + # capture, which the agent has no power to do. + store_agent_content: bool = True + + @classmethod + def from_managed_payload(cls, payload: dict) -> "ServerAgentConfig": + """Parse a complete protocol-v1 policy, failing closed on bad fields.""" + nested_policy = payload.get("policy_snapshot") or payload.get("policy") + merged = {**payload, **nested_policy} if isinstance(nested_policy, dict) else payload + protocol = merged.get("managed_protocol_version", merged.get("version")) + tools = merged.get("tools") + commands = merged.get("commands_denylist") + iterations = merged.get("max_iterations") + context_tokens = merged.get("max_context_tokens") + temperature = merged.get("temperature") + required_text = ("agent_profile", "framework_version", "model", "transport") + if protocol != "1" or any( + not isinstance(merged.get(key), str) or not merged[key].strip() + for key in required_text + ): + raise ValueError("Managed agent policy metadata is incomplete.") + if merged["framework_version"] not in {"code4me2-agent", "code4me-agent"}: + raise ValueError("Managed agent policy selects an unsupported runtime.") + if merged["transport"] != "managed_backend": + raise ValueError("Managed agent policy selects an unsafe transport.") + if commands is None: + raise ValueError( + "Managed agent policy has no commands_denylist: the server is older than " + "this runtime release." + ) + if ( + not isinstance(tools, list) + or not all(isinstance(tool, str) and tool.strip() for tool in tools) + or not isinstance(commands, list) + or not all(isinstance(command, str) and command.strip() for command in commands) + or isinstance(iterations, bool) + or not isinstance(iterations, int) + or iterations < 1 + or isinstance(context_tokens, bool) + or not isinstance(context_tokens, int) + or context_tokens < 1 + or merged.get("approval_policy") not in {"auto", "per_step", "suggestion_only"} + or isinstance(temperature, bool) + or ( + temperature is not None + and ( + not isinstance(temperature, (int, float)) + or not 0.0 <= float(temperature) <= 2.0 + ) + ) + or not isinstance(merged.get("store_agent_content"), bool) + or ( + merged.get("system_prompt") is not None + and not isinstance(merged.get("system_prompt"), str) + ) + or not _valid_optional_timeout(merged.get("command_timeout_seconds")) + ): + raise ValueError("Managed agent policy contains invalid executable settings.") + try: + parse_harness_overrides(merged.get("harness_options"), strict=True) + except ValueError as exc: + raise ValueError( + f"Managed agent policy contains invalid executable settings: {exc}" + ) from None + return cls.from_payload(merged) + + @classmethod + def from_payload(cls, payload: dict) -> "ServerAgentConfig": + """Parse an agent-config response, ignoring anything malformed. + + Every field is validated independently: one bad value costs that + override, not the whole config, because a partially-usable server + config still beats falling back to a local guess. + """ + + nested_policy = payload.get("policy_snapshot") or payload.get("policy") + if isinstance(nested_policy, dict): + payload = {**payload, **nested_policy} + + def _clean_str(key: str) -> str | None: + value = payload.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + def _clean_str_list(key: str) -> list[str] | None: + value = payload.get(key) + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + return None + + def _clean_positive_int(key: str) -> int | None: + value = payload.get(key) + if isinstance(value, bool): + return None + if isinstance(value, int) and value >= 1: + return value + return None + + temperature = payload.get("temperature") + if not isinstance(temperature, (int, float)) or isinstance(temperature, bool): + temperature = None + + command_timeout = payload.get("command_timeout_seconds") + if not _valid_optional_timeout(command_timeout) or command_timeout is None: + command_timeout = None + + return cls( + agent_profile=_clean_str("agent_profile"), + framework_version=_clean_str("framework_version"), + model=_clean_str("model"), + base_url=_clean_str("base_url"), + api_key_ref=_clean_str("api_key_ref"), + commands_denylist=_clean_str_list("commands_denylist"), + tools=_clean_str_list("tools"), + max_iterations=_clean_positive_int("max_iterations"), + max_context_tokens=_clean_positive_int("max_context_tokens"), + approval_policy=_clean_str("approval_policy"), + temperature=float(temperature) if temperature is not None else None, + system_prompt=_clean_str("system_prompt"), + command_timeout_seconds=int(command_timeout) if command_timeout is not None else None, + harness_options=parse_harness_overrides(payload.get("harness_options"), strict=False), + store_agent_content=bool(payload.get("store_agent_content", True)), + ) + + @property + def has_overrides(self) -> bool: + return any( + value is not None + for value in ( + self.model, + self.base_url, + self.api_key_ref, + self.commands_denylist, + self.tools, + self.max_iterations, + self.max_context_tokens, + self.approval_policy, + self.temperature, + self.system_prompt, + self.command_timeout_seconds, + self.harness_options, + ) + ) + + +def _valid_optional_timeout(value: object) -> bool: + if value is None: + return True + return ( + isinstance(value, int) + and not isinstance(value, bool) + and 1 <= value <= 600 + ) + + +@dataclass(frozen=True) +class AgentConfig: + workspace_root: Path + trace_path: Path + session_id: str + tools: list[str] | None = None + raw_capture_enabled: bool = False + upload: UploadConfig = field(default_factory=UploadConfig) + commands: CommandConfig = field(default_factory=CommandConfig) + adapter: AdapterConfig = field(default_factory=AdapterConfig) + allowed_tools: frozenset[str] | None = None + approval_policy: str = "auto" + store_agent_content: bool = True + harness: HarnessOptions = field(default_factory=HarnessOptions) + # Headless runs (benchmarks, scripts): nobody reads or answers the turn, so + # a final message that only announces more work is continued. Never set by + # a study profile; IDE (ACP) sessions keep the default (as gemini-cli does). + autonomous: bool = False + managed_mode: bool = False + managed_request: Callable[[str, str, dict | None], dict] | None = field( + default=None, repr=False, compare=False + ) + + def with_server_overrides( + self, server: ServerAgentConfig, *, backend_url: str | None = None + ) -> "AgentConfig": + """Return a copy with the backend's assigned configuration applied. + + This is where the assigned agent profile actually takes hold of the + runtime. Precedence is deliberate: **server wins**, and the local + config file is only a fallback for anything the server didn't specify. + + An authenticated runtime always relays through the backend. Provider + credentials stay in the backend environment; the local agent holds + only its short-lived ACP bearer token. + """ + from dataclasses import replace as _replace + + # A server-governed session sends only what the profile defines. + provider = _replace(self.adapter.provider, extra_body={}) + + if self.managed_mode and backend_url: + provider = _replace( + provider, + kind="managed_backend", + base_url=str(backend_url).rstrip("/"), + api_key_env="", + auth_headers={}, + ) + elif backend_url: + provider = _replace( + provider, + kind="code4me_backend", + base_url=str(backend_url).rstrip("/"), + ) + elif server.base_url: + provider = _replace( + provider, kind="openai", base_url=server.base_url + ) + + if server.model: + provider = _replace(provider, model=server.model) + if server.api_key_ref and provider.kind not in ("code4me_backend", "managed_backend"): + provider = _replace(provider, api_key_env=server.api_key_ref) + if server.temperature is not None: + provider = _replace(provider, temperature=server.temperature) + + memory_window = self.adapter.memory_window + if server.max_context_tokens is not None: + memory_window = _replace( + memory_window, max_tokens=server.max_context_tokens + ) + + adapter = _replace( + self.adapter, + provider=provider, + memory_window=memory_window, + max_iterations=server.max_iterations or self.adapter.max_iterations, + system_prompt=( + server.system_prompt + if server.system_prompt is not None + else self.adapter.system_prompt + ), + ) + + commands = self.commands + if server.commands_denylist is not None: + commands = _replace(commands, blocked_commands=list(server.commands_denylist)) + if server.command_timeout_seconds is not None: + timeout = float(server.command_timeout_seconds) + commands = _replace( + commands, + timeout_seconds=timeout, + max_timeout_seconds=max(timeout, commands.max_timeout_seconds), + ) + + tools = self.tools + allowed_tools = self.allowed_tools + if server.tools is not None: + tools = list(server.tools) + allowed_tools = frozenset(server.tools) + + return _replace( + self, + adapter=adapter, + commands=commands, + tools=tools, + allowed_tools=allowed_tools, + approval_policy=server.approval_policy or self.approval_policy, + store_agent_content=server.store_agent_content, + harness=self.harness.with_overrides(server.harness_options), + ) + + @classmethod + def from_file(cls, config_path: str | Path) -> "AgentConfig": + path = Path(config_path).expanduser().resolve() + data = json.loads(path.read_text()) + + workspace_root_str = data.get("workspace_root", ".") + workspace_root = Path(str(workspace_root_str)).expanduser() + if not workspace_root.is_absolute(): + workspace_root = Path.cwd() / workspace_root + workspace_root = workspace_root.resolve() + + telemetry = data.get("telemetry", {}) + raw_trace_path = telemetry.get( + "trace_path", data.get("trace_path", ".code4me/acp-trace.jsonl") + ) + trace_path = Path(raw_trace_path).expanduser() + if not trace_path.is_absolute(): + trace_path = workspace_root / trace_path + + upload_data = telemetry.get("upload", {}) + raw_headers = upload_data.get("auth_headers", {}) + auth_headers: dict[str, str] = {} + if isinstance(raw_headers, dict): + auth_headers = { + str(header): str(value) + for header, value in raw_headers.items() + if header and value is not None + } + bearer_token = upload_data.get("bearer_token") + if bearer_token: + auth_headers["Authorization"] = f"Bearer {bearer_token}" + + raw_batch_size = upload_data.get("batch_size", 50) + try: + batch_size = max(1, int(raw_batch_size)) + except (TypeError, ValueError): + batch_size = 50 + + raw_timeout_seconds = upload_data.get("timeout_seconds", 5.0) + try: + timeout_seconds = float(raw_timeout_seconds) + if timeout_seconds <= 0: + timeout_seconds = 5.0 + except (TypeError, ValueError): + timeout_seconds = 5.0 + + raw_ingest_url = upload_data.get("ingest_url") + ingest_url = str(raw_ingest_url).strip() if raw_ingest_url else None + if not ingest_url: + ingest_url = None + + upload = UploadConfig( + enabled=bool(upload_data.get("enabled", False)), + ingest_url=ingest_url, + batch_size=batch_size, + timeout_seconds=timeout_seconds, + auth_headers=auth_headers, + ) + + commands_data = data.get("commands", data.get("command_tools", {})) + raw_denylist = commands_data.get( + "denylist", commands_data.get("blocked_commands", []) + ) + blocked_commands: list[str] = [] + if isinstance(raw_denylist, list): + blocked_commands = [ + str(command).strip() + for command in raw_denylist + if isinstance(command, str) and command.strip() + ] + if "allowlist" in commands_data or "allowlisted_commands" in commands_data: + # Files written before the denylist: the runtime now runs any + # program except blocked ones, so an allowlist no longer applies. + logger.warning( + "%s: commands.allowlist is no longer supported and is ignored; list the " + "programs the agent may not run under commands.denylist.", + path, + ) + + command_timeout_seconds = _positive_float( + commands_data.get("timeout_seconds", 120.0), default=120.0 + ) + max_command_timeout_seconds = _positive_float( + commands_data.get("max_timeout_seconds", 600.0), default=600.0 + ) + + raw_max_output_bytes = commands_data.get("max_output_bytes", 16384) + try: + max_output_bytes = max(1, int(raw_max_output_bytes)) + except (TypeError, ValueError): + max_output_bytes = 16384 + + commands = CommandConfig( + blocked_commands=blocked_commands, + timeout_seconds=command_timeout_seconds, + max_timeout_seconds=max(command_timeout_seconds, max_command_timeout_seconds), + max_output_bytes=max_output_bytes, + ) + + adapter_data = data.get("adapter", {}) + if not isinstance(adapter_data, dict): + adapter_data = {} + + raw_max_iterations = adapter_data.get("max_iterations", 8) + try: + max_iterations = max(1, int(raw_max_iterations)) + except (TypeError, ValueError): + max_iterations = 8 + + memory_window_data = adapter_data.get("memory_window", {}) + if not isinstance(memory_window_data, dict): + memory_window_data = {} + memory_scope = str(memory_window_data.get("scope", "prompt")).strip() + if memory_scope not in {"prompt", "session"}: + memory_scope = "prompt" + memory_strategy = str( + memory_window_data.get("strategy", "last_messages") + ).strip() + if memory_strategy not in {"last_messages", "token_window"}: + memory_strategy = "last_messages" + raw_max_messages = memory_window_data.get("max_messages", 12) + try: + max_messages = max(1, int(raw_max_messages)) + except (TypeError, ValueError): + max_messages = 12 + raw_max_tokens = memory_window_data.get("max_tokens", 32000) + try: + max_tokens = max(1, int(raw_max_tokens)) + except (TypeError, ValueError): + max_tokens = 32000 + + fake_provider_data = adapter_data.get("fake_provider", {}) + if not isinstance(fake_provider_data, dict): + fake_provider_data = {} + raw_script = fake_provider_data.get("script", []) + script = raw_script if isinstance(raw_script, list) else [] + + provider_data = adapter_data.get("provider", {}) + if not isinstance(provider_data, dict): + provider_data = {} + raw_provider_headers = provider_data.get("auth_headers", {}) + provider_auth_headers: dict[str, str] = {} + if isinstance(raw_provider_headers, dict): + provider_auth_headers = { + str(header): str(value) + for header, value in raw_provider_headers.items() + if header and value is not None + } + + adapter = AdapterConfig( + name=str(adapter_data.get("name", "deterministic_echo")).strip() + or "deterministic_echo", + max_iterations=max_iterations, + memory_window=MemoryWindowConfig( + scope=memory_scope, + strategy=memory_strategy, + max_messages=max_messages, + max_tokens=max_tokens, + ), + fake_provider=FakeProviderConfig( + enabled=bool(fake_provider_data.get("enabled", False)), + script=script, + ), + # No provider defaults are invented here. Anything the config file + # omits stays empty and is filled from the backend's agent-config + # after authentication (see AgentConfig.with_server_overrides), so + # the assigned profile — not a local fallback — decides the model + # and endpoint. + provider=OpenAICompatibleProviderConfig( + kind=str(provider_data.get("kind", "code4me_backend")).strip() + or "code4me_backend", + base_url=str(provider_data.get("base_url", "")).strip(), + model=str(provider_data.get("model", "")).strip(), + api_key_env=str(provider_data.get("api_key_env", "")).strip(), + timeout_seconds=_positive_float( + provider_data.get("timeout_seconds", 90.0), + default=90.0, + ), + temperature=( + float(provider_data["temperature"]) + if isinstance(provider_data.get("temperature"), (int, float)) + and not isinstance(provider_data.get("temperature"), bool) + else None + ), + auth_headers=provider_auth_headers, + max_output_tokens=( + int(provider_data["max_output_tokens"]) + if isinstance(provider_data.get("max_output_tokens"), int) + and not isinstance(provider_data.get("max_output_tokens"), bool) + and provider_data["max_output_tokens"] > 0 + else None + ), + request_deadline_seconds=( + _positive_float(provider_data["request_deadline_seconds"], default=0.0) or None + if provider_data.get("request_deadline_seconds") is not None + else None + ), + extra_body=_extra_body(provider_data.get("extra_body")), + ), + ) + + harness_data = data.get("harness_options", adapter_data.get("harness_options")) + harness = HarnessOptions().with_overrides( + parse_harness_overrides(harness_data, strict=False) + ) + + return cls( + workspace_root=workspace_root, + trace_path=trace_path.resolve(), + session_id=data.get("session_id", uuid4().hex), + raw_capture_enabled=bool(telemetry.get("raw_capture_enabled", False)), + upload=upload, + commands=commands, + adapter=adapter, + harness=harness, + autonomous=data.get("autonomous") is True, + ) + + +def _positive_float(value: object, *, default: float) -> float: + try: + parsed = float(value) + if parsed > 0: + return parsed + except (TypeError, ValueError): + pass + return default + + +# Fields the agent itself owns in a request body; extra_body may not replace them. +_RESERVED_REQUEST_FIELDS = frozenset({ + "model", "messages", "tools", "tool_choice", "temperature", "max_tokens", + "max_completion_tokens", "stream", "stream_options", "n", +}) + + +def _extra_body(value: object) -> dict[str, Any]: + """Validated ``provider.extra_body``. Invalid input is an error, not ignored: + dropping, say, an upstream pin would silently change which model answers.""" + if value is None: + return {} + if not isinstance(value, dict) or not all(isinstance(key, str) and key for key in value): + raise ValueError("provider.extra_body must be a JSON object with string keys") + reserved = sorted(_RESERVED_REQUEST_FIELDS.intersection(value)) + if reserved: + raise ValueError(f"provider.extra_body may not set {', '.join(reserved)}") + try: + return json.loads(json.dumps(value)) + except (TypeError, ValueError) as exc: + raise ValueError(f"provider.extra_body must be JSON: {exc}") from exc diff --git a/src/code4me2_agent/echo.py b/src/code4me2_agent/echo.py new file mode 100644 index 0000000..cca76e7 --- /dev/null +++ b/src/code4me2_agent/echo.py @@ -0,0 +1,229 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass +from time import perf_counter +from typing import TYPE_CHECKING +from uuid import uuid4 + +from code4me2_agent.adapters import MemoryWindow, create_agent_adapter +from code4me2_agent.command_tools import WorkspaceCommandTools +from code4me2_agent.file_tools import WorkspaceFileTools +from code4me2_agent.session_state import SessionToolState +from code4me2_agent.telemetry import AgentTelemetryRecorder + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from threading import Event + + from code4me2_agent.config import AgentConfig + from code4me2_agent.events import AgentEventSink + from code4me2_agent.mcp_tools import StdioMcpToolBroker + + +@dataclass(frozen=True) +class EchoPromptResult: + run_id: str + final_response: str + stop_reason: str + run_status: str + thoughts: tuple[str, ...] = () + duration_ms: float = 0.0 + # True when the adapter already streamed the final text to the client. + response_emitted: bool = False + usage: dict[str, int] | None = None + + +class EchoAgentCore: + def __init__( + self, config: AgentConfig, *, event_sink: AgentEventSink | None = None + ) -> None: + self._config = config + self._event_sink = event_sink + self._telemetry = AgentTelemetryRecorder(config) + # What the model has read and what each turn changed: kept for the + # whole chat, across the tool rebuilds every policy refresh causes. + self.session_state = SessionToolState() + self.file_tools = WorkspaceFileTools( + config, telemetry=self._telemetry, change_observer=self.session_state.record_change + ) + self.command_tools = WorkspaceCommandTools(config, telemetry=self._telemetry) + self._session_memory = self._build_session_memory() + self._acp_file_backend = None + self._acp_command_backend = None + self._mcp_tools = None + self._adapter = create_agent_adapter( + config, + telemetry=self._telemetry, + file_tools=self.file_tools, + command_tools=self.command_tools, + event_sink=self._event_sink, + session_state=self.session_state, + ) + + def rebuild_tools( + self, + *, + acp_file_backend: object | None = None, + acp_command_backend: object | None = None, + mcp_tools: StdioMcpToolBroker | None = None, + ) -> None: + self._acp_file_backend = acp_file_backend + self._acp_command_backend = acp_command_backend + self._mcp_tools = mcp_tools + self.file_tools = WorkspaceFileTools( + self._config, + acp_backend=acp_file_backend, + telemetry=self._telemetry, + change_observer=self.session_state.record_change, + ) + self.command_tools = WorkspaceCommandTools( + self._config, + acp_backend=acp_command_backend, + telemetry=self._telemetry, + ) + self.rebuild_adapter(mcp_tools=mcp_tools) + + def apply_config(self, config: AgentConfig) -> None: + """Apply an immutable server policy and rebuild every enforcement point.""" + previous_memory = self.session_memory_snapshot() + self._config = config + self._telemetry.apply_config(config) + # Memory limits are part of the assigned A/B policy. Recreate the + # window so a policy refresh cannot leave an existing session using + # the previous arm's context budget, while retaining its conversation. + self._session_memory = self._build_session_memory() + if self._session_memory is not None and previous_memory is not None: + self._session_memory.replace_messages(previous_memory) + self.rebuild_tools( + acp_file_backend=self._acp_file_backend, + acp_command_backend=self._acp_command_backend, + mcp_tools=self._mcp_tools, + ) + + def rebuild_adapter( + self, + *, + mcp_tools: StdioMcpToolBroker | None = None, + ) -> None: + self._adapter = create_agent_adapter( + self._config, + telemetry=self._telemetry, + file_tools=self.file_tools, + command_tools=self.command_tools, + event_sink=self._event_sink, + mcp_tools=mcp_tools, + session_state=self.session_state, + ) + + def _build_session_memory(self) -> MemoryWindow | None: + memory_config = self._config.adapter.memory_window + if memory_config.scope != "session": + return None + return MemoryWindow( + strategy=memory_config.strategy, + max_messages=memory_config.max_messages, + max_tokens=memory_config.max_tokens, + ) + + def load_session_memory(self, messages: list[dict]) -> None: + if self._session_memory is None: + return + self._session_memory.replace_messages(messages) + + def session_memory_snapshot(self) -> list[dict] | None: + if self._session_memory is None: + return None + return self._session_memory.snapshot() + + def handle_prompt( + self, + prompt: str, + request_id: str | None = None, + message_id: str | None = None, + run_id: str | None = None, + cancellation_event: Event | None = None, + ) -> EchoPromptResult: + started_at = perf_counter() + request_id = request_id or uuid4().hex + run_id = run_id or uuid4().hex + run_start = self._telemetry.record( + event_type="agent.run.started", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={"status": "started"}, + ) + prompt_event = self._telemetry.record( + event_type="agent.request.received", + run_id=run_id, + request_id=request_id, + parent_event_id=run_start["event_id"], + message_id=message_id, + payload={"role": "user", "content": prompt}, + contains_user_prompt=True, + raw_payload={"role": "user", "content": prompt, "message_id": message_id}, + ) + + adapter_result = self._adapter.handle_prompt( + prompt=prompt, + run_id=run_id, + request_id=request_id, + message_id=message_id, + memory=self._session_memory, + cancellation_event=cancellation_event, + ) + final_response = adapter_result.final_response + + response_event = self._telemetry.record( + event_type="agent.response.completed", + run_id=run_id, + request_id=request_id, + parent_event_id=prompt_event["event_id"], + message_id=message_id, + payload={ + "role": "agent", + "content": final_response, + "stop_reason": adapter_result.stop_reason, + }, + metrics={"response_character_count": len(final_response)}, + contains_agent_response=True, + raw_payload={ + "role": "agent", + "content": final_response, + "message_id": message_id, + "stop_reason": adapter_result.stop_reason, + }, + ) + duration_ms = round((perf_counter() - started_at) * 1000, 3) + run_payload: dict = {"status": adapter_result.run_status} + run_metrics: dict = { + "duration_ms": duration_ms, + "prompt_character_count": len(prompt), + "response_character_count": len(final_response), + } + # Optional model calls (self-review, summary) that failed have no + # closing event of their own; the run reports them. + side_call_failures = list(getattr(adapter_result, "side_call_failures", ()) or ()) + if side_call_failures: + run_payload["side_call_failures"] = side_call_failures + run_metrics["failed_side_calls"] = len(side_call_failures) + self._telemetry.record( + event_type="agent.run.completed", + run_id=run_id, + request_id=request_id, + parent_event_id=response_event["event_id"], + payload=run_payload, + metrics=run_metrics, + ) + return EchoPromptResult( + run_id=run_id, + final_response=final_response, + stop_reason=adapter_result.stop_reason, + run_status=adapter_result.run_status, + thoughts=adapter_result.thoughts, + duration_ms=duration_ms, + response_emitted=bool(getattr(adapter_result, "response_emitted", False)), + usage=getattr(adapter_result, "usage", None), + ) diff --git a/src/code4me2_agent/edit_matching.py b/src/code4me2_agent/edit_matching.py new file mode 100644 index 0000000..1b85289 --- /dev/null +++ b/src/code4me2_agent/edit_matching.py @@ -0,0 +1,345 @@ +"""Locating ``old_text`` for replace_text / edit_file. + +Models copy snippets imperfectly: trailing spaces, a wrong base indentation or +a slightly misremembered middle line make an exact search fail and cost a +retry. The chain below tries progressively looser comparisons, but every +strategy must still find exactly one place; a looser strategy is never used to +break a tie that a stricter one found. The strategy that matched is reported so +the caller can surface it (and the diff card shows the exact result). +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from difflib import SequenceMatcher + +EXACT = "exact" +LINE_ENDINGS = "line_endings" +TRAILING_WHITESPACE = "trailing_whitespace" +INDENTATION = "indentation" +WHITESPACE = "whitespace" +BLOCK_ANCHOR = "block_anchor" + +FUZZY_STRATEGIES = frozenset({TRAILING_WHITESPACE, INDENTATION, WHITESPACE, BLOCK_ANCHOR}) + +# Block-anchor matching: minimum lines, allowed line-count drift and the +# similarity the lines between the anchors must reach. +_BLOCK_MIN_LINES = 3 +_BLOCK_SIMILARITY = 0.75 + + +@dataclass(frozen=True) +class EditMatch: + start: int + end: int + replacement: str + strategy: str + line: int + + +@dataclass(frozen=True) +class MatchResult: + matches: tuple[EditMatch, ...] + strategy: str | None + # 1-based lines of every candidate when the search was ambiguous. + candidate_lines: tuple[int, ...] = () + + @property + def count(self) -> int: + return len(self.matches) + + +def find_matches(text: str, old_text: str, new_text: str, *, replace_all: bool = False) -> MatchResult: + """Return the matches of the first strategy that finds any. + + With ``replace_all`` only exact and line-ending matching are used: a fuzzy + comparison applied everywhere could rewrite unrelated code. + """ + exact = _exact(text, old_text, new_text, EXACT) + if exact: + return MatchResult(exact, EXACT, tuple(match.line for match in exact)) + converted = _convert_line_endings(text, old_text, new_text) + if converted is not None: + matches = _exact(text, converted[0], converted[1], LINE_ENDINGS) + if matches: + return MatchResult(matches, LINE_ENDINGS, tuple(match.line for match in matches)) + if replace_all: + return MatchResult((), None) + lines = _Lines(text) + old = _SnippetLines(old_text) + if not old.lines or all(not line.strip() for line in old.lines): + return MatchResult((), None) + new = _SnippetLines(new_text) + for strategy in (TRAILING_WHITESPACE, INDENTATION, WHITESPACE, BLOCK_ANCHOR): + windows = _line_windows(lines, old, strategy) + if not windows: + continue + matches = tuple( + _line_match(lines, old, new, first, last, strategy) for first, last in windows + ) + return MatchResult(matches, strategy, tuple(first + 1 for first, _last in windows)) + return MatchResult((), None) + + +def first_line_hint(text: str, old_text: str) -> str | None: + """Where the first meaningful line of ``old_text`` occurs, for a no-match hint.""" + first = next((line.strip() for line in old_text.splitlines() if line.strip()), "") + if len(first) < 4: + return None + hits = [index + 1 for index, line in enumerate(text.splitlines()) if line.strip() == first] + if not hits: + return None + shown = ", ".join(str(hit) for hit in hits[:5]) + more = f" (and {len(hits) - 5} more)" if len(hits) > 5 else "" + return f"Its first line appears at line {shown}{more}; read that region and copy the text exactly." + + +# --------------------------------------------------------------- exact search + + +def _exact(text: str, old_text: str, new_text: str, strategy: str) -> tuple[EditMatch, ...]: + matches: list[EditMatch] = [] + position = text.find(old_text) + while position != -1: + matches.append( + EditMatch( + start=position, + end=position + len(old_text), + replacement=new_text, + strategy=strategy, + line=text.count("\n", 0, position) + 1, + ) + ) + position = text.find(old_text, position + len(old_text)) + return tuple(matches) + + +def _convert_line_endings(text: str, old_text: str, new_text: str) -> tuple[str, str] | None: + file_crlf = "\r\n" in text + old_crlf = "\r\n" in old_text + if file_crlf and not old_crlf and "\n" in old_text: + return old_text.replace("\n", "\r\n"), new_text.replace("\r\n", "\n").replace("\n", "\r\n") + if not file_crlf and old_crlf: + return old_text.replace("\r\n", "\n"), new_text.replace("\r\n", "\n") + return None + + +# ---------------------------------------------------------- line strategies + + +class _Lines: + """The file split into lines with their character offsets.""" + + def __init__(self, text: str) -> None: + self.text = text + self.lines: list[str] = [] + self.starts: list[int] = [] + self.ends: list[int] = [] # end of line content, before "\r\n" / "\n" + self.eols: list[str] = [] + position = 0 + for raw in text.splitlines(keepends=True): + content = raw.rstrip("\r\n") + self.starts.append(position) + self.ends.append(position + len(content)) + self.eols.append(raw[len(content):]) + self.lines.append(content) + position += len(raw) + + def eol(self) -> str: + for eol in self.eols: + if eol: + return eol + return "\n" + + +class _SnippetLines: + def __init__(self, snippet: str) -> None: + normalized = snippet.replace("\r\n", "\n") + self.trailing_newline = normalized.endswith("\n") + body = normalized[:-1] if self.trailing_newline else normalized + self.lines = body.split("\n") if body else [] + + +def _indent(line: str) -> str: + return line[: len(line) - len(line.lstrip())] + + +def _common_indent(lines: list[str]) -> str: + indents = [_indent(line) for line in lines if line.strip()] + if not indents: + return "" + common = indents[0] + for indent in indents[1:]: + while not indent.startswith(common): + common = common[:-1] + return common + + +def _dedented(lines: list[str]) -> list[str]: + common = _common_indent(lines) + return [line[len(common):].rstrip() if line.strip() else "" for line in lines] + + +def _collapsed(line: str) -> str: + return " ".join(line.split()) + + +def _line_windows(lines: _Lines, old: _SnippetLines, strategy: str) -> list[tuple[int, int]]: + count = len(old.lines) + total = len(lines.lines) + windows: list[tuple[int, int]] = [] + if strategy == BLOCK_ANCHOR: + if count < _BLOCK_MIN_LINES: + return [] + first_anchor = old.lines[0].strip() + last_anchor = old.lines[-1].strip() + if not first_anchor or not last_anchor: + return [] + tolerance = max(1, count // 4) + old_middle = "\n".join(line.strip() for line in old.lines[1:-1]) + for first in range(total): + if lines.lines[first].strip() != first_anchor: + continue + best: tuple[float, int] | None = None + for length in range(max(2, count - tolerance), count + tolerance + 1): + last = first + length - 1 + if last >= total or lines.lines[last].strip() != last_anchor: + continue + middle = "\n".join(line.strip() for line in lines.lines[first + 1 : last]) + ratio = SequenceMatcher(None, old_middle, middle, autojunk=False).ratio() + if ratio >= _BLOCK_SIMILARITY and (best is None or ratio > best[0]): + best = (ratio, last) + if best is not None: + windows.append((first, best[1])) + return windows + if count == 0 or count > total: + return [] + if strategy == TRAILING_WHITESPACE: + target = [line.rstrip() for line in old.lines] + compare = lambda window: [line.rstrip() for line in window] # noqa: E731 + elif strategy == INDENTATION: + target = _dedented(old.lines) + compare = _dedented + elif strategy == WHITESPACE: + target = [_collapsed(line) for line in old.lines] + compare = lambda window: [_collapsed(line) for line in window] # noqa: E731 + else: # pragma: no cover - guarded by the caller + return [] + first = 0 + while first + count <= total: + if compare(lines.lines[first : first + count]) == target: + windows.append((first, first + count - 1)) + first += count + else: + first += 1 + return windows + + +def _line_match( + lines: _Lines, + old: _SnippetLines, + new: _SnippetLines, + first: int, + last: int, + strategy: str, +) -> EditMatch: + matched = lines.lines[first : last + 1] + new_lines = list(new.lines) + if strategy in (INDENTATION, WHITESPACE): + new_lines = _adapt_indentation(new_lines, old.lines, matched) + elif strategy == BLOCK_ANCHOR: + new_lines = _reindent(new_lines, from_indent=_common_indent(old.lines), to_indent=_common_indent(matched)) + eol = lines.eols[first] or lines.eol() + replacement = eol.join(new_lines) + if new.trailing_newline and not old.trailing_newline: + replacement += eol + return EditMatch( + start=lines.starts[first], + end=lines.ends[last], + replacement=replacement, + strategy=strategy, + line=first + 1, + ) + + +def _adapt_indentation(new_lines: list[str], old_lines: list[str], matched: list[str]) -> list[str]: + """Carry the file's indentation over to ``new_text``. + + The snippet's lines and the matched lines pair up one to one, which gives a + map from each indentation the model used to the one the file uses. Lines + of new_text with a mapped indentation take the file's; other lines follow + the pattern the map shows: a constant shift (wrong base indentation) or a + constant width ratio (two-space snippet in a four-space file). + """ + mapping: dict[str, str] = {} + for old_line, file_line in zip(old_lines, matched): + if not old_line.strip() or not file_line.strip(): + continue + source, target = _indent(old_line), _indent(file_line) + if mapping.setdefault(source, target) != target: + return _reindent( + new_lines, from_indent=_common_indent(old_lines), to_indent=_common_indent(matched) + ) + new_indents = {_indent(line) for line in new_lines if line.strip()} + if not new_indents <= set(mapping) and new_indents <= set(mapping.values()): + # new_text is already written in the file's indentation. + return list(new_lines) + from_base, to_base = _common_indent(old_lines), _common_indent(matched) + is_shift = all( + source.startswith(from_base) and target == to_base + source[len(from_base):] + for source, target in mapping.items() + ) + ratio = None if is_shift else _space_ratio(mapping) + adapted: list[str] = [] + for line in new_lines: + if not line.strip(): + adapted.append("") + continue + indent = _indent(line) + body = line[len(indent):] + if indent in mapping: + adapted.append(mapping[indent] + body) + elif ratio is not None and set(indent) <= {" "}: + width = len(indent) * ratio + adapted.append((" " * int(width) if width == int(width) else indent) + body) + elif indent.startswith(from_base): + adapted.append(to_base + indent[len(from_base):] + body) + else: + adapted.append(line) + return adapted + + +def _space_ratio(mapping: dict[str, str]) -> float | None: + ratios = { + len(target) / len(source) + for source, target in mapping.items() + if source and set(source) <= {" "} and set(target) <= {" "} + } + if len(ratios) != 1: + return None + if any(bool(source) != bool(target) for source, target in mapping.items()): + return None + return ratios.pop() + + +def _reindent(lines: list[str], *, from_indent: str, to_indent: str) -> list[str]: + if from_indent == to_indent: + return lines + result: list[str] = [] + for line in lines: + if not line.strip(): + result.append("") + elif line.startswith(from_indent): + result.append(to_indent + line[len(from_indent):]) + else: + # Shallower than the snippet's base: keep its relative shape. + result.append(to_indent + line.lstrip()) + return result + + +_LINE_NUMBER_PREFIX_RE = re.compile(r"^\s*\d+\|") + + +def has_line_number_prefix(snippet: str) -> bool: + return bool(_LINE_NUMBER_PREFIX_RE.match(snippet)) diff --git a/src/code4me2_agent/events.py b/src/code4me2_agent/events.py new file mode 100644 index 0000000..ef7f6b0 --- /dev/null +++ b/src/code4me2_agent/events.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol + + +@dataclass(frozen=True) +class ApprovalDecision: + decision: str + scope: str = "none" + # Set only when the user picked "Revise…": the ACP form's outcome + # ("accept", "decline", "cancel" or "error"), and for an accepted form + # (decision "revised") the indexes of the hunks to keep and what should change. + elicitation_action: str | None = None + kept_hunks: tuple[int, ...] = () + instructions: str = "" + + @property + def accepted(self) -> bool: + return self.decision == "accepted" + + +@dataclass(frozen=True) +class ToolCallEvent: + phase: str + tool_call_id: str + tool_name: str + run_id: str + request_id: str + title: str + kind: str + status: str + path: str | None = None + content_text: str | None = None + diff_old_text: str | None = None + diff_new_text: str | None = None + raw_input: dict[str, Any] | None = None + raw_output: dict[str, Any] | None = None + # Absolute paths for the ACP card. When omitted the sink falls back to ``path``. + locations: tuple[str, ...] | None = None + # Further file diffs of a multi-file change: (absolute path, old text, new text). + extra_diffs: tuple[tuple[str, str | None, str], ...] = () + + +@dataclass(frozen=True) +class ThoughtEvent: + run_id: str + request_id: str + text: str + phase: str | None = None + duration_ms: float | None = None + + +@dataclass(frozen=True) +class AssistantTextEvent: + """Assistant text produced by one model call, shown to the user as it arrives.""" + + run_id: str + request_id: str + message_id: str + text: str + final: bool + iteration: int + + +@dataclass(frozen=True) +class PlanEntrySpec: + content: str + status: str = "pending" + priority: str = "medium" + + +@dataclass(frozen=True) +class PlanEvent: + run_id: str + request_id: str + tool_call_id: str + entries: tuple[PlanEntrySpec, ...] + + +@dataclass(frozen=True) +class UsageEvent: + run_id: str + request_id: str + iteration: int + model: str + prompt_tokens: int + completion_tokens: int + total_tokens: int + turn_total_tokens: int + context_budget_tokens: int + + +class AgentEventSink(Protocol): + def tool_call(self, event: ToolCallEvent) -> None: + ... + + def thought(self, event: ThoughtEvent) -> None: + ... + + def assistant_text(self, event: AssistantTextEvent) -> None: + ... + + def plan(self, event: PlanEvent) -> None: + ... + + def usage(self, event: UsageEvent) -> None: + ... + + +class NoopAgentEventSink: + def tool_call(self, event: ToolCallEvent) -> None: + return + + def thought(self, event: ThoughtEvent) -> None: + return + + def assistant_text(self, event: AssistantTextEvent) -> None: + return + + def plan(self, event: PlanEvent) -> None: + return + + def usage(self, event: UsageEvent) -> None: + return + + +def emit_event(sink: object, method_name: str, event: object) -> None: + """Deliver ``event`` to ``sink.`` when the sink implements it. + + Test doubles and older sinks implement only ``tool_call``/``request_approval``; + newer event kinds are therefore optional on the sink side. + """ + handler = getattr(sink, method_name, None) + if callable(handler): + handler(event) diff --git a/src/code4me2_agent/file_tools.py b/src/code4me2_agent/file_tools.py new file mode 100644 index 0000000..8126e64 --- /dev/null +++ b/src/code4me2_agent/file_tools.py @@ -0,0 +1,2174 @@ +from __future__ import annotations + +import errno +import inspect +import os +import re +import shutil +from dataclasses import dataclass +from difflib import unified_diff +from pathlib import Path +from time import perf_counter +from typing import TYPE_CHECKING, Any, Callable, Iterator, Sequence +from uuid import uuid4 + +from code4me2_agent.acp_utils import capability_value +from code4me2_agent.async_bridge import run_awaitable_blocking +from code4me2_agent.edit_matching import ( + EXACT, + FUZZY_STRATEGIES, + find_matches, + first_line_hint, + has_line_number_prefix, +) +from code4me2_agent.patching import PatchAction, PatchError, PlannedFile, plan_patch +from code4me2_agent.session_state import FileChange +from code4me2_agent.syntax_check import introduced_syntax_problem +from code4me2_agent.telemetry import AgentTelemetryRecorder +from code4me2_agent.tool_errors import ( + DirectoryNotEmptyError, + EditMatchError, + FileTooLargeError, + NotTextFileError, + ToolArgumentError, + ToolError, + ToolFileExistsError, + ToolFileNotFoundError, + WorkspaceBoundaryError, +) + +if TYPE_CHECKING: + from code4me2_agent.config import AgentConfig + + +DEFAULT_READ_LIMIT_LINES = 1000 +MAX_READ_LIMIT_LINES = 5000 +MAX_LINE_CHARS = 2000 +MAX_READ_OUTPUT_BYTES = 24 * 1024 +MAX_FILE_BYTES = 20 * 1024 * 1024 +BINARY_SNIFF_BYTES = 8192 +MAX_GREP_FILE_BYTES = 2 * 1024 * 1024 +MAX_GREP_LINE_CHARS = 500 +# Lines longer than this are not matched at all: they are almost always +# minified assets, and unbounded regex backtracking on them would block the +# worker thread. +MAX_GREP_SCAN_LINE_CHARS = 10_000 +MAX_WALK_ENTRIES = 100_000 +DEFAULT_IGNORED_DIRS: frozenset[str] = frozenset( + { + ".git", + ".hg", + ".svn", + "node_modules", + ".venv", + "venv", + "__pycache__", + "build", + "dist", + "target", + "out", + ".idea", + ".gradle", + ".code4me", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".tox", + } +) + + +@dataclass(frozen=True) +class FileToolLimits: + default_read_limit_lines: int = DEFAULT_READ_LIMIT_LINES + max_read_limit_lines: int = MAX_READ_LIMIT_LINES + max_line_chars: int = MAX_LINE_CHARS + max_read_output_bytes: int = MAX_READ_OUTPUT_BYTES + max_file_bytes: int = MAX_FILE_BYTES + binary_sniff_bytes: int = BINARY_SNIFF_BYTES + max_grep_file_bytes: int = MAX_GREP_FILE_BYTES + max_grep_line_chars: int = MAX_GREP_LINE_CHARS + max_grep_scan_line_chars: int = MAX_GREP_SCAN_LINE_CHARS + max_walk_entries: int = MAX_WALK_ENTRIES + + +@dataclass(frozen=True) +class FileReadResult: + path: str + content: str + backend_type: str + start_line: int = 1 + end_line: int = 0 + total_lines: int = 0 + truncated: bool = False + truncation_reason: str | None = None + next_offset: int | None = None + decoding_errors_replaced: bool = False + + +@dataclass(frozen=True) +class FileWriteResult: + path: str + bytes_written: int + backend_type: str + # Set when the write introduced a Python/JSON/TOML syntax error. + syntax_error: dict[str, object] | None = None + + +@dataclass(frozen=True) +class FileEditResult: + path: str + bytes_written: int + backend_type: str + replacements: int + edits_applied: int + # Per edit, only when an edit matched with a non-exact strategy. + match_strategies: list[str] | None = None + syntax_error: dict[str, object] | None = None + + +@dataclass(frozen=True) +class FilePatchResult: + files: list[dict[str, object]] + backend_type: str + file_count: int + syntax_errors: list[dict[str, object]] | None = None + + +@dataclass(frozen=True) +class FileDeleteResult: + path: str + backend_type: str + was_directory: bool + bytes_removed: int + + +@dataclass(frozen=True) +class FileMoveResult: + source_path: str + destination_path: str + backend_type: str + overwritten: bool + was_directory: bool + + +@dataclass(frozen=True) +class FileListResult: + path: str + entries: list[str] + backend_type: str + file_count: int + directory_count: int + truncated: bool + + +@dataclass(frozen=True) +class FileGlobResult: + pattern: str + path: str + files: list[str] + backend_type: str + file_count: int + truncated: bool + + +@dataclass(frozen=True) +class FileGrepResult: + pattern: str + path: str + output_mode: str + backend_type: str + matches: list[dict[str, object]] | None + files: list[str] | None + counts: list[dict[str, object]] | None + match_count: int + file_count: int + files_searched: int + files_skipped: int + truncated: bool + cancelled: bool = False + + +# Backwards-compatible alias: ``search_files`` now returns grep-shaped results. +FileSearchResult = FileGrepResult + + +@dataclass(frozen=True) +class TextEdit: + old_text: str + new_text: str + replace_all: bool = False + + +@dataclass(frozen=True) +class EditOutcome: + text: str + replacements: int + strategies: tuple[str, ...] + + @property + def fuzzy(self) -> bool: + return any(strategy in FUZZY_STRATEGIES for strategy in self.strategies) + + +class AcpFileSystemBackend: + def __init__( + self, + *, + client: object, + session_id: str, + read_text_file_enabled: bool, + write_text_file_enabled: bool, + async_runner: object | None = None, + ) -> None: + self._client = client + self._session_id = session_id + self._read_text_file_enabled = read_text_file_enabled + self._write_text_file_enabled = write_text_file_enabled + self._async_runner = async_runner + + @property + def read_text_file_enabled(self) -> bool: + return self._read_text_file_enabled + + @property + def write_text_file_enabled(self) -> bool: + return self._write_text_file_enabled + + @property + def session_id(self) -> str: + return self._session_id + + def read_text_file(self, absolute_path: str) -> str: + response = self._run_client_call( + self._client.read_text_file( + path=absolute_path, + session_id=self._session_id, + limit=None, + line=None, + ) + ) + if isinstance(response, dict): + return str(response.get("content", "")) + return str(getattr(response, "content", "")) + + def write_text_file(self, absolute_path: str, content: str) -> None: + self._run_client_call( + self._client.write_text_file( + content=content, + path=absolute_path, + session_id=self._session_id, + ) + ) + + def _run_client_call(self, result: object) -> object: + if not inspect.isawaitable(result): + return result + if self._async_runner is not None: + return self._async_runner.run(result) + return run_awaitable_blocking(result) + + +def build_acp_file_system_backend( + *, + client: object, + session_id: str, + client_capabilities: object | None, + async_runner: object | None = None, +) -> AcpFileSystemBackend | None: + fs_capabilities = capability_value(client_capabilities, "fs") + read_enabled = bool( + capability_value(fs_capabilities, "readTextFile") + or capability_value(fs_capabilities, "read_text_file") + ) + write_enabled = bool( + capability_value(fs_capabilities, "writeTextFile") + or capability_value(fs_capabilities, "write_text_file") + ) + if not read_enabled and not write_enabled: + return None + return AcpFileSystemBackend( + client=client, + session_id=session_id, + read_text_file_enabled=read_enabled, + write_text_file_enabled=write_enabled, + async_runner=async_runner, + ) + + +class WorkspaceFileTools: + def __init__( + self, + config: AgentConfig, + acp_backend: object | None = None, + telemetry: AgentTelemetryRecorder | None = None, + *, + limits: FileToolLimits | None = None, + ignored_dirs: frozenset[str] | set[str] | None = None, + change_observer: Callable[[FileChange], None] | None = None, + ) -> None: + self._config = config + self._acp_backend = acp_backend + self._telemetry = telemetry or AgentTelemetryRecorder(config) + self._limits = limits or FileToolLimits() + self._ignored_dirs = frozenset(ignored_dirs) if ignored_dirs is not None else DEFAULT_IGNORED_DIRS + self._change_observer = change_observer + + @property + def workspace_root(self) -> Path: + return self._config.workspace_root + + @property + def limits(self) -> FileToolLimits: + return self._limits + + # ------------------------------------------------------------------ reading + + def read_text( + self, + path: str, + *, + strict: bool = False, + tool_name: str = "read_file", + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> tuple[str, bool]: + """Return ``(text, decoding_errors_replaced)`` without recording a tool event. + + Used by the registry to build edit previews; ``tool_name`` attributes any + denial or backend-fallback event to the calling tool. ``strict=True`` + rejects undecodable files instead of replacing bad bytes. + """ + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + resolved_path = self._resolve_or_record_denial( + tool_name, path, tool_call_id, run_id, request_id, started_at + ) + text, replaced, _backend = self._read_resolved( + resolved_path, + strict=strict, + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + return text, replaced + + def read_file( + self, + path: str, + *, + offset: int | None = None, + limit: int | None = None, + line_start: int | None = None, + line_end: int | None = None, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileReadResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + # Legacy aliases kept for older profiles/histories. + if offset is None and line_start is not None: + offset = int(line_start) + if limit is None and line_end is not None: + limit = max(1, int(line_end) - (offset or 1) + 1) + offset = 1 if offset is None else int(offset) + if offset < 1: + raise ToolArgumentError("offset must be a positive line number.", field="offset") + if limit is None: + limit = self._limits.default_read_limit_lines + limit = max(1, min(int(limit), self._limits.max_read_limit_lines)) + resolved_path = self._resolve_or_record_denial( + "read_file", path, tool_call_id, run_id, request_id, started_at + ) + relative_path = self._relative_path(resolved_path) + text, replaced, backend_type = self._read_resolved( + resolved_path, + strict=False, + tool_name="read_file", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + rendered = _render_numbered(text, offset=offset, limit=limit, limits=self._limits) + self._record_tool_event( + tool_name="read_file", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + status="completed", + backend_type=backend_type, + started_at=started_at, + extra_payload={ + "start_line": rendered["start_line"], + "end_line": rendered["end_line"], + "total_lines": rendered["total_lines"], + "truncated": rendered["truncated"], + }, + ) + return FileReadResult( + path=relative_path, + content=rendered["content"], + backend_type=backend_type, + start_line=rendered["start_line"], + end_line=rendered["end_line"], + total_lines=rendered["total_lines"], + truncated=rendered["truncated"], + truncation_reason=rendered["truncation_reason"], + next_offset=rendered["next_offset"], + decoding_errors_replaced=replaced, + ) + + # ------------------------------------------------------------------ writing + + def create_file( + self, + path: str, + content: str, + *, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileWriteResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + resolved_path = self._resolve_or_record_denial( + "create_file", path, tool_call_id, run_id, request_id, started_at + ) + relative_path = self._relative_path(resolved_path) + if self._exists_for_create(resolved_path): + raise ToolFileExistsError( + f"File already exists: {relative_path}. Use write_file to overwrite it or " + "replace_text/edit_file to change part of it." + ) + backend_type = self._write_text( + resolved_path, + content, + tool_name="create_file", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + bytes_written = len(content.encode("utf-8")) + self._notify_change(FileChange(relative_path, None, content)) + syntax_error = self._syntax_problem(relative_path, None, content) + self._record_tool_event( + tool_name="create_file", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + status="completed", + backend_type=backend_type, + started_at=started_at, + extra_payload={ + "bytes_written": bytes_written, + "content_capture_mode": self._content_capture_mode, + **_syntax_payload(syntax_error), + }, + raw_payload=self._raw_write_payload(relative_path, "", content), + ) + return FileWriteResult( + path=relative_path, + bytes_written=bytes_written, + backend_type=backend_type, + syntax_error=syntax_error, + ) + + def write_file( + self, + path: str, + content: str, + *, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + tool_name: str = "write_file", + ) -> FileWriteResult: + """Write ``content`` to ``path``, reported as ``tool_name``. + + The registry also writes the hunks a user kept with "Revise…" through + here, reported under the edit tool that proposed them. + """ + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + resolved_path = self._resolve_or_record_denial( + tool_name, path, tool_call_id, run_id, request_id, started_at + ) + relative_path = self._relative_path(resolved_path) + if resolved_path.is_dir(): + raise ToolArgumentError( + f"{relative_path} is a directory, not a file.", field="path" + ) + before, before_known = self._read_before( + resolved_path, + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + backend_type = self._write_text( + resolved_path, + content, + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + bytes_written = len(content.encode("utf-8")) + self._notify_change(FileChange(relative_path, before, content, before_known=before_known)) + syntax_error = self._syntax_problem(relative_path, before, content) + self._record_tool_event( + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + status="completed", + backend_type=backend_type, + started_at=started_at, + extra_payload={ + "bytes_written": bytes_written, + "content_capture_mode": self._content_capture_mode, + **_syntax_payload(syntax_error), + }, + raw_payload=self._raw_write_payload(relative_path, before or "", content), + ) + return FileWriteResult( + path=relative_path, + bytes_written=bytes_written, + backend_type=backend_type, + syntax_error=syntax_error, + ) + + def replace_text( + self, + path: str, + old_text: str, + new_text: str, + *, + replace_all: bool = False, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileEditResult: + return self._apply_edits( + path, + [TextEdit(old_text=old_text, new_text=new_text, replace_all=replace_all)], + tool_name="replace_text", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + ) + + def edit_file( + self, + path: str, + edits: Sequence[TextEdit], + *, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileEditResult: + return self._apply_edits( + path, + list(edits), + tool_name="edit_file", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + ) + + def _apply_edits( + self, + path: str, + edits: Sequence[TextEdit], + *, + tool_name: str, + tool_call_id: str | None, + run_id: str | None, + request_id: str | None, + ) -> FileEditResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + if not edits: + raise ToolArgumentError("edits must contain at least one replacement.", field="edits") + resolved_path = self._resolve_or_record_denial( + tool_name, path, tool_call_id, run_id, request_id, started_at + ) + relative_path = self._relative_path(resolved_path) + current, _replaced, _backend = self._read_resolved( + resolved_path, + strict=True, + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + outcome = apply_text_edits_detailed(current, edits, path=relative_path) + updated, replacements = outcome.text, outcome.replacements + backend_type = self._write_text( + resolved_path, + updated, + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + bytes_written = len(updated.encode("utf-8")) + self._notify_change(FileChange(relative_path, current, updated)) + syntax_error = self._syntax_problem(relative_path, current, updated) + match_strategies = list(outcome.strategies) if outcome.fuzzy else None + extra_payload: dict[str, object] = { + "bytes_written": bytes_written, + "replacements": replacements, + "edits_applied": len(edits), + "content_capture_mode": self._content_capture_mode, + **_syntax_payload(syntax_error), + } + if match_strategies is not None: + extra_payload["match_strategies"] = match_strategies + self._record_tool_event( + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + status="completed", + backend_type=backend_type, + started_at=started_at, + extra_payload=extra_payload, + raw_payload=self._raw_write_payload(relative_path, current, updated), + ) + return FileEditResult( + path=relative_path, + bytes_written=bytes_written, + backend_type=backend_type, + replacements=replacements, + edits_applied=len(edits), + match_strategies=match_strategies, + syntax_error=syntax_error, + ) + + def plan_patch( + self, + actions: list[PatchAction], + *, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> list[PlannedFile]: + """Resolve a parsed patch against the current files without writing. + + Every path is confined to the workspace (a path outside it records the + denial and raises), and the same file named twice in different + spellings is refused: both entries would plan against the original. + Names differing only in case count as the same file, as they are on + case-insensitive filesystems (macOS and Windows defaults). + """ + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + seen: dict[str, str] = {} + for action in actions: + for path in filter(None, (action.path, action.move_to)): + if action.action == "delete": + resolved = self._resolve_lexical_or_record_denial( + "apply_patch", path, tool_call_id, run_id, request_id, started_at + ) + else: + resolved = self._resolve_or_record_denial( + "apply_patch", path, tool_call_id, run_id, request_id, started_at + ) + key = self._file_identity(resolved) + if key in seen: + raise PatchError( + f"{path} and {seen[key]} are the same file; change each file once per patch." + ) + seen[key] = path + + def read(path: str) -> str | None: + resolved = self._resolve_workspace_path(path) + try: + text, _replaced, _backend = self._read_resolved( + resolved, + strict=True, + tool_name="apply_patch", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + except ToolFileNotFoundError: + return None + return text + + return plan_patch(actions, read) + + def apply_patch( + self, + actions: list[PatchAction], + *, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FilePatchResult: + """Apply a parsed V4A patch: every file is resolved before any is written. + + If a write fails part-way, the files already written are restored, so + the patch applies completely or not at all. + """ + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + try: + planned = self.plan_patch( + actions, tool_call_id=tool_call_id, run_id=run_id, request_id=request_id + ) + except PatchError as exc: + raise ToolError(str(exc), code="patch_does_not_apply") from None + backend_type = "local" + files: list[dict[str, object]] = [] + syntax_errors: list[dict[str, object]] = [] + diff_parts: list[str] = [] + changes: list[FileChange] = [] + # [resolved path, text to restore or None to remove, step completed] + undo: list[list[object]] = [] + try: + for item in planned: + source = self._resolve_workspace_path(item.path) + relative = self._relative_path(source) + entry: dict[str, object] = {"path": relative, "action": item.action} + # Each undo entry is registered before its write, so a write + # that fails part-way (after truncating) is restored as well. + if item.action == "delete": + resolved = self._resolve_lexical_path(item.path) + undo.append([resolved, item.old_text, False]) + resolved.unlink() + undo[-1][2] = True + changes.append(FileChange(relative, item.old_text, None)) + entry["removed_lines"] = len((item.old_text or "").splitlines()) + else: + target_path = self._resolve_workspace_path(item.target) + target = self._relative_path(target_path) + undo.append([target_path, item.old_text if item.move_to is None else None, False]) + backend_type = self._write_text( + target_path, + item.new_text or "", + tool_name="apply_patch", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + undo[-1][2] = True + if item.move_to is not None: + moved_from = self._resolve_lexical_path(item.path) + undo.append([moved_from, item.old_text, False]) + moved_from.unlink() + undo[-1][2] = True + entry["move_to"] = target + changes.append(FileChange(relative, item.old_text, None)) + changes.append(FileChange(target, None, item.new_text)) + else: + changes.append(FileChange(relative, item.old_text, item.new_text)) + added, removed = _line_delta(item.old_text or "", item.new_text or "") + entry["added_lines"] = added + entry["removed_lines"] = removed + if item.fuzz: + entry["fuzzy_context"] = True + problem = self._syntax_problem(target, item.old_text, item.new_text or "") + if problem is not None: + syntax_errors.append({"path": target, **problem}) + diff_parts.append( + self._raw_write_payload(relative, item.old_text or "", item.new_text or "")["diff"] + ) + files.append(entry) + except Exception as exc: + restored = self._rollback_patch(undo) + raise ToolError( + f"Applying the patch failed ({exc}); " + + ("the files already changed were restored." if restored else "restoring some files failed."), + code="patch_write_failed", + ) from exc + for change in changes: + self._notify_change(change) + extra_payload: dict[str, object] = { + "file_count": len(files), + "actions": [entry["action"] for entry in files], + "content_capture_mode": self._content_capture_mode, + } + if any(entry.get("fuzzy_context") for entry in files): + extra_payload["fuzzy_context"] = True + if syntax_errors: + extra_payload["syntax_check"] = "failed" + self._record_tool_event( + tool_name="apply_patch", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=str(files[0]["path"]) if files else "", + status="completed", + backend_type=backend_type, + started_at=started_at, + extra_payload=extra_payload, + raw_payload={"diff": "".join(diff_parts)}, + ) + return FilePatchResult( + files=files, + backend_type=backend_type, + file_count=len(files), + syntax_errors=syntax_errors or None, + ) + + def _file_identity(self, resolved: Path) -> str: + """A key under which two spellings of one file compare equal. + + Existing files are identified by device and inode (case variants on a + case-insensitive filesystem, hard links); some filesystems report + inode 0 for everything, and then the path is used. A path that does + not exist yet is compared case-insensitively on every filesystem: two + new files differing only in case cannot be told apart where the + filesystem ignores case, so such a patch is refused rather than + silently merged. + """ + try: + info = os.lstat(resolved) + if info.st_ino: + return f"inode:{info.st_dev}:{info.st_ino}" + return f"path:{resolved}" + except OSError: + return f"new:{str(resolved).casefold()}" + + def _rollback_patch(self, undo: list[list[object]]) -> bool: + """Put every touched file back; False only when one could not be restored. + + Completed steps are always restored (through the same backend that + wrote them, so an IDE buffer is restored too). Only the step that was + in flight when the failure happened may have changed nothing; it is + skipped when the file already holds its original state, read through + the same view the patch was planned against. + """ + restored = True + for path, text, done in reversed(undo): + try: + if not done and self._holds_original(path, text): + continue + if text is None: + if path.exists() or path.is_symlink(): + path.unlink() + continue + self._write_text( + path, + text, + tool_name="apply_patch", + tool_call_id=uuid4().hex, + run_id=uuid4().hex, + request_id=uuid4().hex, + started_at=perf_counter(), + ) + except Exception: # noqa: BLE001 - keep restoring the rest + restored = False + return restored + + def _holds_original(self, path: Path, text: object) -> bool: + """Whether a step that failed left ``path`` unchanged in every view. + + A write goes to the IDE (ACP) first and falls back to the disk, so + both must still hold the original before the step can be skipped. + """ + views: list[str | None] = [] + try: + current, _replaced, _backend = self._read_resolved( + path, + strict=True, + tool_name="apply_patch", + tool_call_id=uuid4().hex, + run_id=uuid4().hex, + request_id=uuid4().hex, + started_at=perf_counter(), + ) + views.append(current) + except ToolFileNotFoundError: + views.append(None) + except Exception: # noqa: BLE001 - unknown state: restore it + return False + if self._acp_read_text_file() is not None: + try: + local, _replaced = self._read_local_text(path, strict=True) + views.append(local) + except ToolFileNotFoundError: + views.append(None) + except Exception: # noqa: BLE001 + return False + return all(view == text for view in views) + + def delete_file( + self, + path: str, + *, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileDeleteResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + resolved_path = self._resolve_lexical_or_record_denial( + "delete_file", path, tool_call_id, run_id, request_id, started_at + ) + relative_path = self._relative_path(resolved_path) + if resolved_path == self._config.workspace_root.resolve(): + raise ToolArgumentError("Refusing to delete the workspace root.", field="path") + is_symlink = resolved_path.is_symlink() + if not resolved_path.exists() and not is_symlink: + raise ToolFileNotFoundError( + f"File not found: {relative_path}. Use glob_files or list_files to find the right path." + ) + was_directory = resolved_path.is_dir() and not is_symlink + before = "" + before_known = False + bytes_removed = 0 + if was_directory: + if any(resolved_path.iterdir()): + raise DirectoryNotEmptyError( + f"Directory is not empty: {relative_path}. Delete its contents first." + ) + resolved_path.rmdir() + elif is_symlink: + # Remove the link itself; its target is untouched. + resolved_path.unlink() + else: + try: + bytes_removed = resolved_path.stat().st_size + before, replaced = self._read_local_text(resolved_path, strict=False) + before_known = not replaced + except (NotTextFileError, FileTooLargeError, OSError): + before = "" + resolved_path.unlink() + self._notify_change( + FileChange(relative_path, before if before_known else None, None, before_known=before_known) + ) + self._record_tool_event( + tool_name="delete_file", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + status="completed", + backend_type="local", + started_at=started_at, + extra_payload={ + "was_directory": was_directory, + "bytes_removed": bytes_removed, + "content_capture_mode": self._content_capture_mode, + }, + raw_payload=self._raw_write_payload(relative_path, before, ""), + ) + return FileDeleteResult( + path=relative_path, + backend_type="local", + was_directory=was_directory, + bytes_removed=bytes_removed, + ) + + def move_file( + self, + source_path: str, + destination_path: str, + *, + overwrite: bool = False, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileMoveResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + source = self._resolve_lexical_or_record_denial( + "move_file", source_path, tool_call_id, run_id, request_id, started_at + ) + destination = self._resolve_lexical_or_record_denial( + "move_file", destination_path, tool_call_id, run_id, request_id, started_at + ) + source_rel = self._relative_path(source) + destination_rel = self._relative_path(destination) + workspace_root = self._config.workspace_root.resolve() + if source == workspace_root: + raise ToolArgumentError("Refusing to move the workspace root.", field="source_path") + if not source.exists() and not source.is_symlink(): + raise ToolFileNotFoundError( + f"File not found: {source_rel}. Use glob_files or list_files to find the right path." + ) + if source == destination: + raise ToolArgumentError( + "source_path and destination_path are the same.", field="destination_path" + ) + was_directory = source.is_dir() and not source.is_symlink() + if was_directory and (destination == source or source in destination.parents): + raise ToolArgumentError( + "destination_path is inside the directory being moved.", field="destination_path" + ) + moved_text = self._checkpoint_text(source) if not was_directory else None + overwritten = False + if destination.exists() or destination.is_symlink(): + if destination.is_dir() and not destination.is_symlink(): + raise ToolArgumentError( + f"Destination is an existing directory: {destination_rel}. Overwriting " + "directories is not supported; choose a new path.", + field="destination_path", + ) + if not overwrite: + raise ToolFileExistsError( + f"Destination already exists: {destination_rel}. Pass overwrite=true to replace it." + ) + if was_directory: + raise ToolArgumentError( + "A directory cannot overwrite an existing file.", field="destination_path" + ) + overwritten = True + replaced_text = self._checkpoint_text(destination) if overwritten else None + destination.parent.mkdir(parents=True, exist_ok=True) + try: + os.replace(source, destination) + except OSError as exc: + if exc.errno == errno.EXDEV: + shutil.move(str(source), str(destination)) + else: + raise + movable = not was_directory and moved_text is not None + self._notify_change(FileChange(source_rel, moved_text, None, before_known=movable)) + self._notify_change( + FileChange( + destination_rel, + replaced_text, + moved_text, + before_known=movable and (not overwritten or replaced_text is not None), + ) + ) + self._record_tool_event( + tool_name="move_file", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=source_rel, + status="completed", + backend_type="local", + started_at=started_at, + extra_payload={ + "destination_path": destination_rel, + "overwritten": overwritten, + "was_directory": was_directory, + }, + ) + return FileMoveResult( + source_path=source_rel, + destination_path=destination_rel, + backend_type="local", + overwritten=overwritten, + was_directory=was_directory, + ) + + # ------------------------------------------------------------- discovery + + def list_files( + self, + path: str = ".", + *, + recursive: bool = False, + max_depth: int | None = None, + max_results: int = 500, + include_ignored: bool = False, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileListResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + resolved_path = self._resolve_or_record_denial( + "list_files", path, tool_call_id, run_id, request_id, started_at + ) + relative_path = self._display_path(resolved_path) + if not resolved_path.exists(): + raise ToolFileNotFoundError(f"Directory not found: {relative_path}.") + if not resolved_path.is_dir(): + raise ToolArgumentError( + f"{relative_path} is a file, not a directory. Use read_file to read it.", + field="path", + ) + max_results = max(1, int(max_results)) + depth_limit = (max_depth if recursive else 1) + entries: list[str] = [] + file_count = 0 + directory_count = 0 + truncated = False + try: + for candidate, is_dir in self._walk( + resolved_path, max_depth=depth_limit, include_ignored=include_ignored + ): + if len(entries) >= max_results: + truncated = True + break + rel = self._relative_path(candidate) + if is_dir: + directory_count += 1 + entries.append(rel + "/") + else: + file_count += 1 + entries.append(rel) + except _WalkLimitReached: + truncated = True + self._record_tool_event( + tool_name="list_files", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + status="completed", + backend_type="local", + started_at=started_at, + extra_payload={ + "path_count": len(entries), + "recursive": bool(recursive), + "truncated": truncated, + }, + ) + return FileListResult( + path=relative_path, + entries=entries, + backend_type="local", + file_count=file_count, + directory_count=directory_count, + truncated=truncated, + ) + + def glob_files( + self, + pattern: str, + *, + path: str = ".", + max_results: int = 500, + include_ignored: bool = False, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileGlobResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + matcher = glob_to_regex(pattern) + resolved_path = self._resolve_or_record_denial( + "glob_files", path, tool_call_id, run_id, request_id, started_at + ) + relative_path = self._display_path(resolved_path) + if not resolved_path.is_dir(): + raise ToolArgumentError( + f"{relative_path} is not a directory.", field="path" + ) + max_results = max(1, int(max_results)) + matches: list[str] = [] + truncated = False + try: + for candidate, is_dir in self._walk( + resolved_path, max_depth=None, include_ignored=include_ignored + ): + if is_dir: + continue + relative_to_base = candidate.relative_to(resolved_path).as_posix() + if matcher.match(relative_to_base): + matches.append(self._relative_path(candidate)) + except _WalkLimitReached: + truncated = True + matches.sort() + if len(matches) > max_results: + matches = matches[:max_results] + truncated = True + self._record_tool_event( + tool_name="glob_files", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + status="completed", + backend_type="local", + started_at=started_at, + extra_payload={ + "pattern": pattern, + "file_count": len(matches), + "truncated": truncated, + }, + ) + return FileGlobResult( + pattern=pattern, + path=relative_path, + files=matches, + backend_type="local", + file_count=len(matches), + truncated=truncated, + ) + + def grep_files( + self, + pattern: str, + *, + path: str = ".", + glob: str | None = None, + case_insensitive: bool = False, + context_lines: int = 0, + max_results: int = 200, + output_mode: str = "content", + include_ignored: bool = False, + cancellation_event: Any | None = None, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileGrepResult: + return self._grep( + pattern, + path=path, + glob=glob, + case_insensitive=case_insensitive, + context_lines=context_lines, + max_results=max_results, + output_mode=output_mode, + include_ignored=include_ignored, + tool_name="grep_files", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + literal_query=None, + cancellation_event=cancellation_event, + ) + + def search_files( + self, + query: str, + *, + path: str = ".", + cancellation_event: Any | None = None, + tool_call_id: str | None = None, + run_id: str | None = None, + request_id: str | None = None, + ) -> FileGrepResult: + """Legacy literal search kept for frozen profiles; grep-shaped result.""" + return self._grep( + re.escape(query), + path=path, + glob=None, + case_insensitive=False, + context_lines=0, + max_results=200, + output_mode="content", + include_ignored=False, + tool_name="search_files", + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + literal_query=query, + cancellation_event=cancellation_event, + ) + + def _grep( + self, + pattern: str, + *, + path: str, + glob: str | None, + case_insensitive: bool, + context_lines: int, + max_results: int, + output_mode: str, + include_ignored: bool, + tool_name: str, + tool_call_id: str | None, + run_id: str | None, + request_id: str | None, + literal_query: str | None, + cancellation_event: Any | None = None, + ) -> FileGrepResult: + started_at = perf_counter() + tool_call_id = tool_call_id or uuid4().hex + run_id = run_id or uuid4().hex + request_id = request_id or uuid4().hex + if output_mode not in {"content", "files_with_matches", "count"}: + raise ToolArgumentError( + "output_mode must be one of content, files_with_matches, count.", + field="output_mode", + ) + if not pattern: + raise ToolArgumentError("pattern must not be empty.", field="pattern") + try: + regex = re.compile(pattern, re.IGNORECASE if case_insensitive else 0) + except re.error as exc: + raise ToolArgumentError( + f"invalid regular expression: {exc}. Escape special characters for literal text.", + field="pattern", + ) from None + glob_matcher = glob_to_regex(glob) if glob else None + context_lines = max(0, min(int(context_lines), 10)) + max_results = max(1, int(max_results)) + resolved_path = self._resolve_or_record_denial( + tool_name, path, tool_call_id, run_id, request_id, started_at + ) + relative_path = self._display_path(resolved_path) + if not resolved_path.exists(): + raise ToolFileNotFoundError(f"Path not found: {relative_path}.") + + matches: list[dict[str, object]] = [] + files_with_matches: list[str] = [] + counts: list[dict[str, object]] = [] + match_count = 0 + files_searched = 0 + files_skipped = 0 + truncated = False + cancelled = False + scan_limit = self._limits.max_grep_scan_line_chars + + def candidates() -> Iterator[Path]: + if resolved_path.is_file(): + yield resolved_path + return + for candidate, is_dir in self._walk( + resolved_path, max_depth=None, include_ignored=include_ignored + ): + if not is_dir: + yield candidate + + try: + for candidate in candidates(): + if cancellation_event is not None and cancellation_event.is_set(): + cancelled = True + truncated = True + break + if glob_matcher is not None: + base = resolved_path if resolved_path.is_dir() else resolved_path.parent + if not glob_matcher.match(candidate.relative_to(base).as_posix()): + continue + lines = self._grep_read_lines(candidate) + if lines is None: + files_skipped += 1 + continue + files_searched += 1 + rel = self._relative_path(candidate) + file_matches = 0 + for line_number, line in enumerate(lines, start=1): + if len(line) > scan_limit or not regex.search(line): + continue + file_matches += 1 + match_count += 1 + if output_mode == "content": + entry: dict[str, object] = { + "path": rel, + "line_number": line_number, + "line": _cut_line(line, self._limits.max_grep_line_chars), + } + if context_lines: + before_start = max(0, line_number - 1 - context_lines) + entry["context_before"] = [ + _cut_line(item, self._limits.max_grep_line_chars) + for item in lines[before_start : line_number - 1] + ] + entry["context_after"] = [ + _cut_line(item, self._limits.max_grep_line_chars) + for item in lines[line_number : line_number + context_lines] + ] + matches.append(entry) + if len(matches) >= max_results: + truncated = True + break + if file_matches: + files_with_matches.append(rel) + counts.append({"path": rel, "count": file_matches}) + if truncated: + break + if output_mode != "content" and len(files_with_matches) >= max_results: + truncated = True + break + except _WalkLimitReached: + truncated = True + + extra_payload: dict[str, object] = { + "pattern": pattern, + "output_mode": output_mode, + "match_count": match_count, + "file_count": len(files_with_matches), + "files_searched": files_searched, + "files_skipped": files_skipped, + "truncated": truncated, + "cancelled": cancelled, + } + if glob: + extra_payload["glob"] = glob + if literal_query is not None: + extra_payload["query"] = literal_query + self._record_tool_event( + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + status="completed", + backend_type="local", + started_at=started_at, + extra_payload=extra_payload, + ) + return FileGrepResult( + pattern=literal_query if literal_query is not None else pattern, + path=relative_path, + output_mode=output_mode, + backend_type="local", + matches=matches if output_mode == "content" else None, + files=files_with_matches if output_mode == "files_with_matches" else None, + counts=counts if output_mode == "count" else None, + match_count=match_count, + file_count=len(files_with_matches), + files_searched=files_searched, + files_skipped=files_skipped, + truncated=truncated, + cancelled=cancelled, + ) + + def _grep_read_lines(self, candidate: Path) -> list[str] | None: + try: + if candidate.stat().st_size > self._limits.max_grep_file_bytes: + return None + data = candidate.read_bytes() + except OSError: + return None + if b"\x00" in data[: self._limits.binary_sniff_bytes]: + return None + return split_lines(data.decode("utf-8", errors="replace")) + + def _walk( + self, + root: Path, + *, + max_depth: int | None, + include_ignored: bool, + ) -> Iterator[tuple[Path, bool]]: + return _walk( + root, + max_depth=max_depth, + include_ignored=include_ignored, + ignored_dirs=self._ignored_dirs, + max_entries=self._limits.max_walk_entries, + ) + + # ------------------------------------------------------------- backends + + def _read_resolved( + self, + resolved_path: Path, + *, + strict: bool, + tool_name: str, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> tuple[str, bool, str]: + relative_path = self._relative_path(resolved_path) + read_text_file = self._acp_read_text_file() + if read_text_file is not None: + try: + content = str(read_text_file(str(resolved_path))) + return content, False, "acp" + except Exception as exc: # noqa: BLE001 + self._record_acp_backend_fallback( + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=relative_path, + started_at=started_at, + error_message=str(exc), + operation="read_text_file", + ) + text, replaced = self._read_local_text(resolved_path, strict=strict) + return text, replaced, "local" + + def _read_local_text(self, resolved_path: Path, *, strict: bool) -> tuple[str, bool]: + relative_path = self._relative_path(resolved_path) + try: + size = resolved_path.stat().st_size + except FileNotFoundError: + raise ToolFileNotFoundError( + f"File not found: {relative_path}. Use glob_files or list_files to find the right path." + ) from None + if resolved_path.is_dir(): + raise ToolArgumentError( + f"{relative_path} is a directory, not a file. Use list_files to see its contents.", + field="path", + ) + if size > self._limits.max_file_bytes: + raise FileTooLargeError( + f"{relative_path} is {size} bytes, above the {self._limits.max_file_bytes} byte limit." + ) + data = resolved_path.read_bytes() + if b"\x00" in data[: self._limits.binary_sniff_bytes]: + raise NotTextFileError( + f"{relative_path} looks like a binary file and cannot be read as text." + ) + try: + return data.decode("utf-8"), False + except UnicodeDecodeError: + if strict: + raise NotTextFileError( + f"{relative_path} is not valid UTF-8 text; editing it is not supported." + ) from None + return data.decode("utf-8", errors="replace"), True + + def _read_before( + self, + resolved_path: Path, + *, + tool_name: str, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> tuple[str | None, bool]: + """``(text, known)``: ``(None, True)`` for a missing file, ``known`` false when unreadable.""" + try: + text, replaced, _backend = self._read_resolved( + resolved_path, + strict=False, + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + started_at=started_at, + ) + return text, not replaced + except ToolFileNotFoundError: + return None, True + except (NotTextFileError, FileTooLargeError, ToolArgumentError): + return None, False + + def _write_text( + self, + resolved_path: Path, + content: str, + *, + tool_name: str, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> str: + write_text_file = self._acp_write_text_file() + if write_text_file is not None: + try: + write_text_file(str(resolved_path), content) + return "acp" + except Exception as exc: # noqa: BLE001 + self._record_acp_backend_fallback( + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=self._relative_path(resolved_path), + started_at=started_at, + error_message=str(exc), + operation="write_text_file", + ) + resolved_path.parent.mkdir(parents=True, exist_ok=True) + # Bytes, not text mode: Windows text mode would turn each "\n" of a CRLF + # file into "\r\n" and double every carriage return. + resolved_path.write_bytes(content.encode("utf-8")) + return "local" + + def _exists_for_create(self, resolved_path: Path) -> bool: + if resolved_path.exists() or resolved_path.is_symlink(): + return True + read_text_file = self._acp_read_text_file() + if read_text_file is None: + return False + try: + content = read_text_file(str(resolved_path)) + except Exception: # noqa: BLE001 + return False + return bool(str(content)) + + def _acp_read_text_file(self) -> Callable[[str], str] | None: + if not bool(getattr(self._acp_backend, "read_text_file_enabled", False)): + return None + operation = getattr(self._acp_backend, "read_text_file", None) + if callable(operation): + return operation + return None + + def _acp_write_text_file(self) -> Callable[[str, str], None] | None: + if not bool(getattr(self._acp_backend, "write_text_file_enabled", False)): + return None + operation = getattr(self._acp_backend, "write_text_file", None) + if callable(operation): + return operation + return None + + # ------------------------------------------------------------- checkpoints + + def restore_text(self, path: str, content: str) -> None: + """Write ``content`` for an undo, without a tool event or change record.""" + resolved_path = self._resolve_workspace_path(path) + self._write_text( + resolved_path, + content, + tool_name="undo", + tool_call_id=uuid4().hex, + run_id=uuid4().hex, + request_id=uuid4().hex, + started_at=perf_counter(), + ) + + def discard_file(self, path: str) -> None: + """Remove a file the agent created, for an undo, without a tool event.""" + resolved_path = self._resolve_lexical_path(path) + if resolved_path.is_dir() and not resolved_path.is_symlink(): + raise ToolArgumentError(f"{path} is a directory.", field="path") + resolved_path.unlink() + + def _notify_change(self, change: FileChange) -> None: + observer = self._change_observer + if observer is None: + return + try: + observer(change) + except Exception: # noqa: BLE001 - checkpoints must never break an edit + pass + + def _syntax_problem(self, path: str, before: str | None, after: str) -> dict[str, object] | None: + harness = getattr(self._config, "harness", None) + if harness is not None and not getattr(harness, "syntax_check", True): + return None + try: + problem = introduced_syntax_problem(path, before, after) + except Exception: # noqa: BLE001 - a checker bug must never fail the edit + return None + return problem.as_result() if problem is not None else None + + def _checkpoint_text(self, resolved_path: Path) -> str | None: + if self._change_observer is None: + return None + try: + text, replaced = self._read_local_text(resolved_path, strict=False) + except (ToolFileNotFoundError, NotTextFileError, FileTooLargeError, ToolArgumentError, OSError): + return None + return None if replaced else text + + # ------------------------------------------------------------- helpers + + def _resolve_workspace_path(self, path: str) -> Path: + workspace_root = self._config.workspace_root.resolve() + requested_path = Path(str(path)).expanduser() + if not requested_path.is_absolute(): + requested_path = workspace_root / requested_path + resolved_path = requested_path.resolve() + if resolved_path != workspace_root and workspace_root not in resolved_path.parents: + raise WorkspaceBoundaryError( + f"Path is outside the workspace root: {path}. Use workspace-relative paths." + ) + return resolved_path + + def _resolve_lexical_path(self, path: str) -> Path: + """Confine ``path`` without following a symlink at its last component. + + Used by delete/move so a symlink inside the workspace is removed or + renamed itself rather than its target. Both the link's own location + and (for links) its target must stay inside the workspace. + """ + workspace_root = self._config.workspace_root.resolve() + requested_path = Path(str(path)).expanduser() + if not requested_path.is_absolute(): + requested_path = workspace_root / requested_path + if not requested_path.name or requested_path.name in {".", ".."}: + return self._resolve_workspace_path(path) + lexical = requested_path.parent.resolve() / requested_path.name + if lexical != workspace_root and workspace_root not in lexical.parents: + raise WorkspaceBoundaryError( + f"Path is outside the workspace root: {path}. Use workspace-relative paths." + ) + if lexical.is_symlink(): + target = lexical.resolve() + if target != workspace_root and workspace_root not in target.parents: + raise WorkspaceBoundaryError( + f"Symbolic link points outside the workspace root: {path}." + ) + return lexical + + def _resolve_lexical_or_record_denial( + self, + tool_name: str, + path: str, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> Path: + try: + return self._resolve_lexical_path(path) + except PermissionError: + self._record_tool_event( + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=str(path), + status="denied", + backend_type="local", + started_at=started_at, + denial_reason="outside_workspace_root", + ) + raise + + def _resolve_or_record_denial( + self, + tool_name: str, + path: str, + tool_call_id: str, + run_id: str, + request_id: str, + started_at: float, + ) -> Path: + try: + return self._resolve_workspace_path(path) + except PermissionError: + self._record_tool_event( + tool_name=tool_name, + tool_call_id=tool_call_id, + run_id=run_id, + request_id=request_id, + path=str(path), + status="denied", + backend_type="local", + started_at=started_at, + denial_reason="outside_workspace_root", + ) + raise + + def _relative_path(self, path: Path) -> str: + root = self._config.workspace_root + try: + return path.relative_to(root).as_posix() + except ValueError: + try: + return path.relative_to(root.resolve()).as_posix() + except ValueError: + return path.as_posix() + + def _display_path(self, resolved_path: Path) -> str: + if resolved_path == self._config.workspace_root.resolve(): + return "." + return self._relative_path(resolved_path) + + @property + def _content_capture_mode(self) -> str: + return "raw" if self._config.raw_capture_enabled else "redacted" + + def _raw_write_payload(self, relative_path: str, before: str, after: str) -> dict[str, str]: + return { + "content": after, + "diff": "".join( + unified_diff( + before.splitlines(keepends=True), + after.splitlines(keepends=True), + fromfile=relative_path, + tofile=relative_path, + ) + ), + } + + def _record_tool_event( + self, + *, + tool_name: str, + tool_call_id: str, + run_id: str, + request_id: str, + path: str, + status: str, + backend_type: str, + started_at: float, + denial_reason: str | None = None, + extra_payload: dict[str, object] | None = None, + raw_payload: dict[str, object] | None = None, + ) -> None: + event_type = "agent.tool.denied" if status == "denied" else "agent.tool.completed" + payload = { + "tool_name": tool_name, + "tool_call_id": tool_call_id, + "backend_type": backend_type, + "duration_ms": round((perf_counter() - started_at) * 1000, 3), + "status": status, + "path": path, + } + payload.update(extra_payload or {}) + if denial_reason is not None: + payload["denial_reason"] = denial_reason + self._telemetry.record( + event_type=event_type, + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload=payload, + raw_payload=raw_payload, + ) + + def _record_acp_backend_fallback( + self, + *, + tool_name: str, + tool_call_id: str, + run_id: str, + request_id: str, + path: str, + started_at: float, + error_message: str, + operation: str, + ) -> None: + self._telemetry.record( + event_type="agent.acp.backend_fallback", + run_id=run_id, + request_id=request_id, + parent_event_id=None, + payload={ + "tool_name": tool_name, + "tool_call_id": tool_call_id, + "path": path, + "backend_type": "acp", + "fallback_backend_type": "local", + "acp_session_id": getattr(self._acp_backend, "session_id", None), + "operation": operation, + "duration_ms": round((perf_counter() - started_at) * 1000, 3), + "error_message": error_message, + }, + ) + + +# ---------------------------------------------------------------- pure helpers + + +class _WalkLimitReached(Exception): + pass + + +def _walk( + root: Path, + *, + max_depth: int | None, + include_ignored: bool, + ignored_dirs: frozenset[str], + max_entries: int, +) -> Iterator[tuple[Path, bool]]: + """Yield ``(path, is_directory)`` depth-first with children sorted by name. + + Ignored directory names are pruned at any depth unless ``include_ignored``; + symbolic links are never followed or listed, so a walk cannot escape the + workspace. Raises ``_WalkLimitReached`` after ``max_entries`` entries. + """ + remaining = [max_entries] + + def visit(directory: Path, depth: int) -> Iterator[tuple[Path, bool]]: + try: + with os.scandir(directory) as scanner: + children = sorted(scanner, key=lambda item: item.name) + except (PermissionError, FileNotFoundError, NotADirectoryError): + return + for child in children: + try: + if child.is_symlink(): + continue + is_dir = child.is_dir(follow_symlinks=False) + except OSError: + continue + remaining[0] -= 1 + if remaining[0] < 0: + raise _WalkLimitReached() + child_path = Path(child.path) + if is_dir: + if not include_ignored and child.name in ignored_dirs: + continue + yield child_path, True + if max_depth is None or depth + 1 < max_depth: + yield from visit(child_path, depth + 1) + else: + yield child_path, False + + yield from visit(root, 0) + + +def glob_to_regex(pattern: str) -> re.Pattern[str]: + """Translate a glob (``*``, ``?``, ``[...]``, ``**``) into an anchored regex over posix paths.""" + if not isinstance(pattern, str) or not pattern.strip(): + raise ToolArgumentError("pattern must be a non-empty glob.", field="pattern") + normalized = pattern.strip().replace("\\", "/") + if normalized.startswith("./"): + normalized = normalized[2:] + if ( + normalized.startswith("/") + or normalized.startswith("~") + or ".." in normalized.split("/") + or (len(normalized) > 1 and normalized[1] == ":") + ): + raise ToolArgumentError( + "pattern must be a relative glob without '..' segments.", field="pattern" + ) + parts: list[str] = [] + index = 0 + length = len(normalized) + while index < length: + char = normalized[index] + if char == "*": + if normalized.startswith("**/", index): + parts.append("(?:.*/)?") + index += 3 + continue + if normalized.startswith("**", index): + parts.append(".*") + index += 2 + continue + parts.append("[^/]*") + index += 1 + continue + if char == "?": + parts.append("[^/]") + index += 1 + continue + if char == "[": + closing = normalized.find("]", index + 1) + if closing == -1: + parts.append(re.escape(char)) + index += 1 + continue + body = normalized[index + 1 : closing] + if body.startswith("!"): + body = "^" + body[1:] + body = body.replace("\\", "\\\\") + parts.append(f"[{body}]") + index = closing + 1 + continue + parts.append(re.escape(char)) + index += 1 + return re.compile("^" + "".join(parts) + "$") + + +def split_lines(text: str) -> list[str]: + """Split on ``\n`` only (like editors), dropping the empty tail after a final newline.""" + if text == "": + return [] + lines = text.split("\n") + if lines[-1] == "" and text.endswith("\n"): + lines.pop() + return [line[:-1] if line.endswith("\r") else line for line in lines] + + +def _cut_line(line: str, max_chars: int) -> str: + if len(line) <= max_chars: + return line + return line[:max_chars] + "…" + + +def _render_numbered( + text: str, + *, + offset: int, + limit: int, + limits: FileToolLimits, +) -> dict[str, Any]: + lines = split_lines(text) + total_lines = len(lines) + if total_lines == 0: + return { + "content": "", + "start_line": 0, + "end_line": 0, + "total_lines": 0, + "truncated": False, + "truncation_reason": None, + "next_offset": None, + } + if offset > total_lines: + raise ToolArgumentError( + f"offset {offset} is past the end of the file (total_lines={total_lines}).", + field="offset", + ) + selected = lines[offset - 1 : offset - 1 + limit] + rendered_lines: list[str] = [] + byte_total = 0 + truncation_reason: str | None = None + end_line = offset - 1 + for position, line in enumerate(selected): + line_number = offset + position + if len(line) > limits.max_line_chars: + line = line[: limits.max_line_chars] + "…[line truncated]" + rendered = f"{line_number}|{line}" + byte_total += len(rendered.encode("utf-8")) + 1 + if byte_total > limits.max_read_output_bytes and rendered_lines: + truncation_reason = "bytes" + break + rendered_lines.append(rendered) + end_line = line_number + if truncation_reason is None and end_line < total_lines: + truncation_reason = "limit" + next_offset = None + if truncation_reason is not None: + next_offset = end_line + 1 + rendered_lines.append( + f"[truncated: showing lines {offset}-{end_line} of {total_lines}; " + f"call read_file with offset={next_offset} to continue]" + ) + return { + "content": "\n".join(rendered_lines), + "start_line": offset, + "end_line": end_line, + "total_lines": total_lines, + "truncated": truncation_reason is not None, + "truncation_reason": truncation_reason, + "next_offset": next_offset, + } + + +_WHITESPACE_RE = re.compile(r"\s+") + + +def apply_text_edits( + text: str, + edits: Sequence[TextEdit], + *, + path: str, +) -> tuple[str, int]: + """Apply edits in order; returns ``(new_text, replacements)``.""" + outcome = apply_text_edits_detailed(text, edits, path=path) + return outcome.text, outcome.replacements + + +def apply_text_edits_detailed( + text: str, + edits: Sequence[TextEdit], + *, + path: str, +) -> EditOutcome: + """Apply edits in order, reporting which matching strategy each one used. + + Every edit must match exactly once unless ``replace_all`` (see + ``edit_matching`` for the strategy chain). Nothing is returned partially + applied: the first failing edit raises and the caller writes nothing. + """ + edit_count = len(edits) + current = text + replacements = 0 + strategies: list[str] = [] + for index, edit in enumerate(edits): + prefix = f"Edit {index + 1} of {edit_count}: " if edit_count > 1 else "" + field_prefix = f"edits[{index}]." if edit_count > 1 else "" + if not isinstance(edit.old_text, str) or edit.old_text == "": + raise ToolArgumentError( + f"{prefix}old_text must not be empty.", field=f"{field_prefix}old_text" + ) + if edit.old_text == edit.new_text: + raise ToolArgumentError( + f"{prefix}old_text and new_text are identical; nothing to change.", + field=f"{field_prefix}new_text", + ) + found = find_matches(current, edit.old_text, edit.new_text, replace_all=edit.replace_all) + if found.count == 0: + raise EditMatchError( + f"{prefix}old_text was not found in {path}. {_no_match_hint(current, edit.old_text)}", + code="edit_no_match", + match_count=0, + edit_index=index, + edit_count=edit_count, + ) + if found.count > 1 and not edit.replace_all: + lines = ", ".join(str(line) for line in found.candidate_lines[:10]) + loosened = ( + "" + if found.strategy in (EXACT, "line_endings") + else f" when {_STRATEGY_DESCRIPTIONS.get(found.strategy or '', 'compared loosely')}" + ) + raise EditMatchError( + f"{prefix}old_text matches {found.count} locations in {path}{loosened} (lines {lines}); " + "include more surrounding lines so it matches exactly once, or set replace_all=true.", + code="edit_ambiguous", + match_count=found.count, + edit_index=index, + edit_count=edit_count, + ) + for match in sorted(found.matches, key=lambda item: item.start, reverse=True): + current = current[: match.start] + match.replacement + current[match.end :] + replacements += found.count + strategies.append(found.strategy or EXACT) + return EditOutcome(text=current, replacements=replacements, strategies=tuple(strategies)) + + +_STRATEGY_DESCRIPTIONS = { + "trailing_whitespace": "ignoring trailing whitespace", + "indentation": "ignoring indentation", + "whitespace": "ignoring whitespace", + "block_anchor": "matching the first and last lines", +} + + +def _line_delta(before: str, after: str) -> tuple[int, int]: + added = removed = 0 + for line in unified_diff(before.splitlines(), after.splitlines(), lineterm="", n=0): + if line.startswith("+") and not line.startswith("+++"): + added += 1 + elif line.startswith("-") and not line.startswith("---"): + removed += 1 + return added, removed + + +def describe_strategy(strategy: str) -> str: + return _STRATEGY_DESCRIPTIONS.get(strategy, strategy) + + +def _syntax_payload(syntax_error: dict[str, object] | None) -> dict[str, object]: + if syntax_error is None: + return {} + return {"syntax_check": "failed", "syntax_error_line": syntax_error.get("line")} + + +def _no_match_hint(text: str, old_text: str) -> str: + if has_line_number_prefix(old_text): + return ( + "old_text starts with the line-number prefix from read_file (e.g. '12|'); " + "remove the prefix and retry." + ) + collapsed_old = _WHITESPACE_RE.sub(" ", old_text).strip() + if collapsed_old and collapsed_old in _WHITESPACE_RE.sub(" ", text): + return ( + "A match exists that differs only in whitespace or line breaks; copy the exact text " + "from read_file." + ) + located = first_line_hint(text, old_text) + if located: + return located + return "Call read_file and copy the exact current text." diff --git a/src/code4me2_agent/hunks.py b/src/code4me2_agent/hunks.py new file mode 100644 index 0000000..068d7f8 --- /dev/null +++ b/src/code4me2_agent/hunks.py @@ -0,0 +1,160 @@ +"""Hunks of a proposed text change, for the "Revise…" approval form. + +``diff_hunks`` splits a change into the hunks a unified diff with ``context`` +lines would show; ``apply_hunks`` applies only some of them to the old text. +Lines are split after each ``\\n`` with their endings kept (the numbering the +file tools and editors use), so CRLF endings and a missing final newline +survive, and both functions are deterministic. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from difflib import SequenceMatcher +from functools import cached_property +from typing import Iterable + +#: The most hunks a "Revise…" form lists; a change with more (or with fewer +#: than two) gets a form that asks for instructions only. +MAX_REVISE_HUNKS = 20 + +_LABEL_TEXT_CHARS = 60 + +_Opcode = tuple[str, int, int, int, int] + + +@dataclass(frozen=True) +class Hunk: + """One group of nearby changed lines (context lines excluded). + + Line numbers are 1-based and inclusive: ``old_start``..``old_end`` in the + current text (``old_end`` is ``old_start - 1`` when the hunk only inserts + before ``old_start``) and ``new_start``..``new_end`` in the proposed text. + ``removed``/``added`` count the changed lines. + """ + + index: int + old_start: int + old_end: int + new_start: int + new_end: int + removed: int + added: int + label: str + + +def diff_hunks(old: str, new: str, context: int = 3) -> list[Hunk]: + """The hunks of the change from ``old`` to ``new``, in file order.""" + old_lines, new_lines = _split_lines(old), _split_lines(new) + _opcodes, groups = _diff(old_lines, new_lines, context) + return [ + _hunk(index, changes, old_lines, new_lines) for index, changes in enumerate(groups) + ] + + +def apply_hunks(old: str, new: str, keep: Iterable[int], context: int = 3) -> str: + """``old`` with only the changes of the hunks in ``keep`` applied. + + ``keep`` holds ``Hunk.index`` values of ``diff_hunks(old, new, context)``; + other values are ignored. Keeping every hunk gives ``new``, keeping none + gives ``old``. + """ + old_lines, new_lines = _split_lines(old), _split_lines(new) + opcodes, groups = _diff(old_lines, new_lines, context) + kept = set(keep) + owner = {change: index for index, changes in enumerate(groups) for change in changes} + merged: list[str] = [] + for opcode in opcodes: + tag, i1, i2, j1, j2 = opcode + if tag != "equal" and owner.get(opcode) in kept: + merged.extend(new_lines[j1:j2]) + else: + merged.extend(old_lines[i1:i2]) + return "".join(merged) + + +@dataclass +class RevisionOffer: + """What "Revise…" offers on one approval request. + + A single-file text change (``path`` with its previewed ``old_text`` and + ``new_text``; ``old_text`` is None for a new file) lets the user keep some + of its hunks; without one (commands, deletes, moves, multi-file patches) + the form asks for instructions only. Hunks are computed on first use, so + an approval the user simply allows or rejects costs no diff. + """ + + path: str | None = None + old_text: str | None = None + new_text: str | None = None + + @cached_property + def hunks(self) -> tuple[Hunk, ...]: + if self.path is None or self.new_text is None: + return () + return tuple(diff_hunks(self.old_text or "", self.new_text)) + + @property + def selectable_hunks(self) -> tuple[Hunk, ...]: + """The hunks the form lists: only for a change of 2 to ``MAX_REVISE_HUNKS``.""" + hunks = self.hunks + return hunks if 2 <= len(hunks) <= MAX_REVISE_HUNKS else () + + +def _split_lines(text: str) -> list[str]: + """``text`` split after each ``\\n``, endings kept (``"".join`` restores it).""" + parts = text.split("\n") + tail = parts.pop() + lines = [part + "\n" for part in parts] + if tail: + lines.append(tail) + return lines + + +def _diff( + old_lines: list[str], new_lines: list[str], context: int +) -> tuple[list[_Opcode], list[list[_Opcode]]]: + """All opcodes, and the changed (non-equal) opcodes of each hunk.""" + matcher = SequenceMatcher(None, old_lines, new_lines, autojunk=False) + # get_grouped_opcodes trims the leading and trailing equal opcodes of the + # matcher's cached list in place: take the full list first. + opcodes = list(matcher.get_opcodes()) + groups = [ + [change for change in group if change[0] != "equal"] + for group in matcher.get_grouped_opcodes(context) + ] + return opcodes, [changes for changes in groups if changes] + + +def _hunk(index: int, changes: list[_Opcode], old_lines: list[str], new_lines: list[str]) -> Hunk: + first, last = changes[0], changes[-1] + old_start, old_end = first[1] + 1, last[2] + removed = sum(i2 - i1 for _tag, i1, i2, _j1, _j2 in changes) + added = sum(j2 - j1 for _tag, _i1, _i2, j1, j2 in changes) + lines = f"L{old_start}" if old_end <= old_start else f"L{old_start}–{old_end}" + label = f"{lines} −{removed} +{added}" + text = _first_text( + [line for _tag, _i1, _i2, j1, j2 in changes for line in new_lines[j1:j2]] + + [line for _tag, i1, i2, _j1, _j2 in changes for line in old_lines[i1:i2]] + ) + if text: + label += f" · {text}" + return Hunk( + index=index, + old_start=old_start, + old_end=old_end, + new_start=first[3] + 1, + new_end=last[4], + removed=removed, + added=added, + label=label, + ) + + +def _first_text(lines: list[str]) -> str: + """The first non-blank line (added lines come first), whitespace collapsed.""" + for line in lines: + text = " ".join(line.split()) + if text: + return text if len(text) <= _LABEL_TEXT_CHARS else text[: _LABEL_TEXT_CHARS - 1] + "…" + return "" diff --git a/src/code4me2_agent/mcp_tools.py b/src/code4me2_agent/mcp_tools.py new file mode 100644 index 0000000..477dc69 --- /dev/null +++ b/src/code4me2_agent/mcp_tools.py @@ -0,0 +1,802 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +import math +import re +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from threading import Event, Thread +from typing import TYPE_CHECKING, Any, Callable, Literal +from urllib.parse import urlsplit + +import anyio +import httpx2 +from mcp import Client, StdioServerParameters +from mcp.client.sse import sse_client +from mcp.client.streamable_http import streamable_http_client + +from code4me2_agent.acp_utils import capability_value + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterable + from pathlib import Path + +logger = logging.getLogger(__name__) + +McpTransport = Literal["stdio", "http", "sse", "unsupported"] +McpToolAccess = Literal["read", "execute", "edit", "other"] + +_MCP_STARTUP_TIMEOUT_SECONDS = 30.0 +_MCP_CALL_TIMEOUT_SECONDS = 120.0 +_MCP_SHUTDOWN_TIMEOUT_SECONDS = 10.0 +# Timeout for HTTP connects and plain request/response exchanges. +_MCP_HTTP_TIMEOUT_SECONDS = 30.0 +# Streamable HTTP responses may be held open as event streams while a tool runs. +_MCP_HTTP_READ_TIMEOUT_SECONDS = 300.0 +# An SSE session carries every response on one long-lived stream; a read timeout +# there would end an idle session, so it is only a last-resort bound. +_MCP_SSE_READ_TIMEOUT_SECONDS = 24 * 60 * 60.0 +_MAX_TOOL_LIST_PAGES = 100 +_MAX_TOOLS_PER_SERVER = 40 +_MAX_ERROR_LENGTH = 200 +_MIN_REDACTED_SECRET_LENGTH = 3 +_OPENAI_TOOL_NAME_LIMIT = 64 +_UNSAFE_TOOL_NAME_CHARACTER_RE = re.compile(r"[^A-Za-z0-9_-]+") +_URL_IN_TEXT_RE = re.compile(r"[A-Za-z][A-Za-z0-9+.\-]*://[^\s'\"<>]+") +_HEADER_NAME_RE = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$") + +# A server is the IntelliJ MCP server when its tool list contains at least three +# of these names; for that server only the curated tools below are exposed. +_INTELLIJ_SIGNATURE_TOOLS = frozenset( + { + "get_file_problems", + "get_run_configurations", + "execute_run_configuration", + "search_in_files_by_text", + "get_symbol_info", + "rename_refactoring", + "build_project", + "lint_files", + "reformat_file", + "get_project_modules", + } +) +_INTELLIJ_SIGNATURE_MINIMUM = 3 +_INTELLIJ_TOOL_ACCESS: dict[str, McpToolAccess] = { + **dict.fromkeys( + ( + "get_file_problems", + "lint_files", + "search_symbol", + "get_symbol_info", + "analyze_calls", + "get_run_configurations", + "get_project_problems", + "git_status", + "get_project_modules", + "get_project_dependencies", + ), + "read", + ), + **dict.fromkeys(("execute_run_configuration", "build_project"), "execute"), + **dict.fromkeys(("rename_refactoring", "reformat_file"), "edit"), +} + + +@dataclass(frozen=True) +class _McpTool: + public_name: str + server_name: str + remote_name: str + definition: dict[str, Any] + access: McpToolAccess = "other" + + +@dataclass(frozen=True) +class _ServerSpec: + """A validated ACP MCP entry. Never log it: it carries headers and env values.""" + + name: str + transport: McpTransport + index: int = 0 + stdio_parameters: StdioServerParameters | None = field(default=None, repr=False) + url: str = field(default="", repr=False) + headers: dict[str, str] = field(default_factory=dict, repr=False) + secrets: tuple[str, ...] = field(default=(), repr=False) + + def client_target(self) -> object: + if self.transport == "stdio": + return self.stdio_parameters + if self.transport == "http": + return _streamable_http_transport(self.url, self.headers) + return sse_client( + self.url, + headers=dict(self.headers), + timeout=_MCP_HTTP_TIMEOUT_SECONDS, + sse_read_timeout=_MCP_SSE_READ_TIMEOUT_SECONDS, + ) + + +@dataclass(frozen=True) +class _StartupOutcome: + client: Client | None = None + tools: list[Any] = field(default_factory=list) + error: str | None = None + + +class _ServerConfigError(ValueError): + """An ACP MCP entry that cannot be used; the message never contains values.""" + + +class McpServerStartupError(RuntimeError): + """No ACP-provided MCP server could be used; ``failures`` says why for each one.""" + + def __init__(self, failures: list[dict[str, str]]) -> None: + self.failures = [dict(item) for item in failures] + summary = "; ".join( + f"{item['name'] or ''} ({item['transport']}): {item['error']}" + for item in self.failures + ) + super().__init__( + f"No ACP-provided MCP server could be started: {summary or 'no server entries'}" + ) + + +def _legacy_mcp_client(target: object) -> Client: + """Build a client that uses the pre-2026 ``initialize`` handshake. + + ``target`` is a ``StdioServerParameters`` for stdio entries and a transport + async context manager for streamable HTTP and SSE entries. + """ + + return Client(target, mode="legacy") # type: ignore[arg-type] + + +@asynccontextmanager +async def _streamable_http_transport(url: str, headers: dict[str, str]) -> AsyncIterator[Any]: + # The HTTP client's default headers are sent on every request of the session. + async with httpx2.AsyncClient( + headers=dict(headers), + timeout=httpx2.Timeout(_MCP_HTTP_TIMEOUT_SECONDS, read=_MCP_HTTP_READ_TIMEOUT_SECONDS), + ) as http_client: + async with streamable_http_client(url, http_client=http_client) as streams: + yield streams + + +class StdioMcpToolBroker: + """Own ACP-provided MCP clients and expose their tools to the synchronous agent loop. + + Entries may use stdio, streamable HTTP or SSE (see ``mcp_server_transport``). + Each server connects in its own task with its own startup deadline: a server + that cannot be started, connected or listed is skipped and reported through + ``failures`` while the others stay usable. + + MCP clients use AnyIO task groups whose context must be exited by the task that + entered it. A dedicated owner thread keeps that lifecycle intact while the + model/tool loop, which already runs in a worker thread, submits tool calls to + the owner's event loop. + """ + + def __init__( + self, + servers: list[object], + *, + cwd: Path, + client_factory: Callable[[Any], Client] = _legacy_mcp_client, + startup_timeout_seconds: float = _MCP_STARTUP_TIMEOUT_SECONDS, + ) -> None: + self._servers = list(servers) + self._cwd = cwd + self._client_factory = client_factory + self._startup_timeout_seconds = float(startup_timeout_seconds) + # Leave headroom inside the overall startup bound to publish the servers + # that did connect before ``open`` stops waiting. + self._connect_timeout_seconds = max( + 0.05, + self._startup_timeout_seconds - min(1.0, self._startup_timeout_seconds / 10), + ) + self._ready = Event() + self._closed = Event() + self._close_flag = Event() + self._loop: asyncio.AbstractEventLoop | None = None + self._close_requested: asyncio.Event | None = None + self._startup_error: BaseException | None = None + self._tools: dict[str, _McpTool] = {} + self._clients: dict[str, Client] = {} + self._failures: list[tuple[int, dict[str, str]]] = [] + self._connected: list[dict[str, Any]] = [] + self._unavailable: set[str] = set() + self._startup_scopes: dict[str, anyio.CancelScope] = {} + self._thread = Thread( + target=self._thread_main, + name="code4me2-mcp", + daemon=True, + ) + + @classmethod + def open( + cls, + servers: list[object], + *, + cwd: Path, + client_factory: Callable[[Any], Client] = _legacy_mcp_client, + startup_timeout_seconds: float = _MCP_STARTUP_TIMEOUT_SECONDS, + ) -> StdioMcpToolBroker | None: + """Connect every entry and return a broker, or None when ``servers`` is empty. + + Servers that fail are skipped and listed in ``failures``. When not a + single server connects, the broker is closed and ``McpServerStartupError`` + (a ``RuntimeError`` carrying ``failures``) is raised instead of returning + an empty broker. ``TimeoutError`` is raised only if the owner thread misses + the overall startup bound. + """ + + if not servers: + return None + broker = cls( + servers, + cwd=cwd, + client_factory=client_factory, + startup_timeout_seconds=startup_timeout_seconds, + ) + broker._thread.start() + if not broker._ready.wait(timeout=startup_timeout_seconds): + broker.close() + raise TimeoutError("Timed out while starting ACP-provided MCP servers.") + if broker._startup_error is not None: + broker.close() + raise RuntimeError( + "Could not start ACP-provided MCP servers: " + f"{type(broker._startup_error).__name__}" + ) from broker._startup_error + if not broker._clients: + failures = broker.failures + broker.close() + raise McpServerStartupError(failures) + return broker + + @property + def failures(self) -> list[dict[str, str]]: + """Servers skipped at startup as ``{"name", "transport", "error"}`` (log-safe).""" + + return [dict(item) for _index, item in sorted(self._failures, key=lambda pair: pair[0])] + + @property + def connected_servers(self) -> list[dict[str, Any]]: + """Connected servers as ``{"name", "transport", "tool_count", "dropped_tool_count", "intellij"}``.""" + + return [dict(item) for item in self._connected] + + def definitions(self) -> list[dict[str, Any]]: + return [tool.definition for tool in self._tools.values()] + + def has_tool(self, name: str) -> bool: + return name in self._tools + + def tool_access(self, name: str) -> McpToolAccess: + """``read``/``execute``/``edit`` for curated IntelliJ tools, ``other`` for any other name.""" + + tool = self._tools.get(name) + return tool.access if tool is not None else "other" + + def is_read_only(self, name: str) -> bool: + return self.tool_access(name) == "read" + + def execute(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + tool = self._tools.get(name) + if tool is None: + raise KeyError(f"Unknown MCP tool: {name}") + loop = self._loop + if ( + loop is None + or loop.is_closed() + or self._closed.is_set() + or tool.server_name in self._unavailable + ): + raise RuntimeError(f"MCP server {tool.server_name!r} is no longer available.") + future = asyncio.run_coroutine_threadsafe( + self._call_tool(tool, arguments), + loop, + ) + try: + return future.result(timeout=_MCP_CALL_TIMEOUT_SECONDS) + except TimeoutError: + future.cancel() + raise TimeoutError( + f"MCP tool {tool.remote_name!r} on server {tool.server_name!r} timed out." + ) from None + + def close(self) -> None: + # Set before reading the loop: an owner loop that is not running yet sees + # the flag once its startup ends instead of waiting for a signal forever. + self._close_flag.set() + if not self._thread.is_alive(): + return + loop = self._loop + close_requested = self._close_requested + if loop is not None and close_requested is not None and not loop.is_closed(): + loop.call_soon_threadsafe(close_requested.set) + self._thread.join(timeout=5.0) + if self._thread.is_alive(): + logger.warning("An ACP-provided MCP server did not stop within five seconds.") + + def _thread_main(self) -> None: + try: + asyncio.run(self._run()) + except BaseException as exc: # noqa: BLE001 + self._startup_error = exc + self._ready.set() + finally: + self._closed.set() + + async def _run(self) -> None: + self._loop = asyncio.get_running_loop() + self._close_requested = asyncio.Event() + stop_servers = asyncio.Event() + specs = self._resolve_specs() + outcomes: dict[str, asyncio.Future[_StartupOutcome]] = {} + tasks: list[asyncio.Task[None]] = [] + try: + for spec in specs: + outcome: asyncio.Future[_StartupOutcome] = self._loop.create_future() + outcomes[spec.name] = outcome + tasks.append( + asyncio.create_task( + self._serve(spec, outcome, stop_servers), + name=f"code4me2-mcp-{spec.transport}", + ) + ) + if outcomes: + await asyncio.wait( + list(outcomes.values()), timeout=self._connect_timeout_seconds + ) + self._publish(specs, outcomes) + self._ready.set() + if not self._close_flag.is_set(): + await self._close_requested.wait() + finally: + stop_servers.set() + for scope in list(self._startup_scopes.values()): + scope.cancel() + if tasks: + done, _pending = await asyncio.wait( + tasks, timeout=_MCP_SHUTDOWN_TIMEOUT_SECONDS + ) + for task in done: + if not task.cancelled(): + task.exception() + + def _resolve_specs(self) -> list[_ServerSpec]: + specs: list[_ServerSpec] = [] + names: set[str] = set() + for index, server in enumerate(self._servers): + name = _server_name(server) + transport = mcp_server_transport(server) + try: + spec = _server_spec( + server, index=index, name=name, transport=transport, cwd=self._cwd + ) + except _ServerConfigError as exc: + self._record_failure(index, name, transport, str(exc)) + continue + if name in names: + self._record_failure(index, name, transport, "duplicate MCP server name") + continue + names.add(name) + specs.append(spec) + if any(spec.transport in ("http", "sse") for spec in specs): + _quiet_http_request_logs() + return specs + + async def _serve( + self, + spec: _ServerSpec, + outcome: asyncio.Future[_StartupOutcome], + stop_servers: asyncio.Event, + ) -> None: + """Connect one server, report the outcome, then hold it open until close.""" + + connected = False + scope = anyio.CancelScope( + deadline=anyio.current_time() + self._connect_timeout_seconds + ) + self._startup_scopes[spec.name] = scope + try: + with scope: + async with self._client_factory(spec.client_target()) as client: + tools = await _list_all_tools(client) + scope.deadline = math.inf + self._startup_scopes.pop(spec.name, None) + if outcome.done(): + return + connected = True + outcome.set_result(_StartupOutcome(client=client, tools=tools)) + await stop_servers.wait() + except Exception as exc: # noqa: BLE001 - one server must not stop the others + error = _describe_error(exc, spec.secrets) + if not connected: + if not outcome.done(): + outcome.set_result(_StartupOutcome(error=error)) + return + self._unavailable.add(spec.name) + if stop_servers.is_set(): + logger.debug( + "ACP-provided MCP server %r (%s) closed with: %s", + spec.name, + spec.transport, + error, + ) + else: + logger.warning( + "ACP-provided MCP server %r (%s) stopped: %s", + spec.name, + spec.transport, + error, + ) + return + finally: + self._startup_scopes.pop(spec.name, None) + if not connected and not outcome.done(): + outcome.set_result(_StartupOutcome(error=self._timeout_error())) + + def _publish( + self, + specs: list[_ServerSpec], + outcomes: dict[str, asyncio.Future[_StartupOutcome]], + ) -> None: + for spec in specs: + outcome = outcomes[spec.name] + if not outcome.done(): + outcome.set_result(_StartupOutcome(error=self._timeout_error())) + scope = self._startup_scopes.get(spec.name) + if scope is not None: + scope.cancel() + result = outcome.result() + if result.client is None: + self._record_failure( + spec.index, spec.name, spec.transport, result.error or "unknown error" + ) + continue + self._clients[spec.name] = result.client + self._register_tools(spec, result.tools) + + def _register_tools(self, spec: _ServerSpec, tools: list[Any]) -> None: + listed = [tool for tool in tools if str(getattr(tool, "name", "") or "")] + remote_names = {str(tool.name) for tool in listed} + intellij = len(remote_names & _INTELLIJ_SIGNATURE_TOOLS) >= _INTELLIJ_SIGNATURE_MINIMUM + if intellij: + selected = [tool for tool in listed if str(tool.name) in _INTELLIJ_TOOL_ACCESS] + logger.info( + "MCP server %r (%s) looks like the IntelliJ MCP server; exposing %d curated " + "tool(s) and dropping %d.", + spec.name, + spec.transport, + len(selected), + len(listed) - len(selected), + ) + else: + selected = listed + selected = sorted(selected, key=lambda tool: str(tool.name)) + if len(selected) > _MAX_TOOLS_PER_SERVER: + logger.warning( + "MCP server %r (%s) offers %d tools; keeping the first %d by name and " + "dropping: %s", + spec.name, + spec.transport, + len(selected), + _MAX_TOOLS_PER_SERVER, + ", ".join(str(tool.name) for tool in selected[_MAX_TOOLS_PER_SERVER:]), + ) + selected = selected[:_MAX_TOOLS_PER_SERVER] + exposed = 0 + for tool in selected: + remote_name = str(tool.name) + public_name = _public_tool_name(spec.name, remote_name) + if public_name in self._tools: + logger.warning( + "Skipping MCP tool %r on server %r: its public name %r is already taken.", + remote_name, + spec.name, + public_name, + ) + continue + parameters = dict(tool.input_schema or {"type": "object"}) + description = str(tool.description or tool.title or remote_name) + self._tools[public_name] = _McpTool( + public_name=public_name, + server_name=spec.name, + remote_name=remote_name, + definition={ + "type": "function", + "function": { + "name": public_name, + "description": f"MCP server {spec.name}: {description}", + "parameters": parameters, + }, + }, + access=_INTELLIJ_TOOL_ACCESS.get(remote_name, "other") if intellij else "other", + ) + exposed += 1 + self._connected.append( + { + "name": spec.name, + "transport": spec.transport, + "tool_count": exposed, + "dropped_tool_count": len(listed) - exposed, + "intellij": intellij, + } + ) + + def _record_failure(self, index: int, name: str, transport: str, error: str) -> None: + self._failures.append((index, {"name": name, "transport": transport, "error": error})) + logger.warning( + "Skipping ACP-provided MCP server %r (%s): %s", name or "", transport, error + ) + + def _timeout_error(self) -> str: + return f"timed out after {self._connect_timeout_seconds:g}s while connecting" + + async def _call_tool( + self, + tool: _McpTool, + arguments: dict[str, Any], + ) -> dict[str, Any]: + client = self._clients.get(tool.server_name) + if client is None or tool.server_name in self._unavailable: + raise RuntimeError(f"MCP server {tool.server_name!r} is no longer available.") + result = await client.call_tool(tool.remote_name, arguments) + payload = result.model_dump(mode="json", by_alias=True, exclude_none=True) + return { + "status": "failed" if result.is_error else "completed", + "server_name": tool.server_name, + "tool_name": tool.remote_name, + "content": payload.get("content", []), + "structured_content": payload.get("structuredContent"), + } + + +McpToolBroker = StdioMcpToolBroker + + +async def _list_all_tools(client: Client) -> list[Any]: + tools: list[Any] = [] + cursor: str | None = None + for _page in range(_MAX_TOOL_LIST_PAGES): + result = await client.list_tools(cursor=cursor) + tools.extend(result.tools) + cursor = result.next_cursor + if cursor is None: + break + return tools + + +def mcp_server_transport(server: object) -> McpTransport: + """Classify an ACP ``session/new`` MCP entry (pydantic model or dict). + + stdio when ``command`` is set; http when ``type == "http"`` or the entry is an + ACP HTTP server model; sse when ``type == "sse"`` or an ACP SSE server model; + anything else (ACP-transport servers, bare URLs) is unsupported. + """ + + if capability_value(server, "command"): + return "stdio" + kind = capability_value(server, "type") + kind_text = kind.strip().lower() if isinstance(kind, str) else "" + model_names = {cls.__name__ for cls in type(server).__mro__} + if kind_text == "http" or model_names & {"McpServerHttp", "HttpMcpServer"}: + return "http" + if kind_text == "sse" or model_names & {"McpServerSse", "SseMcpServer"}: + return "sse" + return "unsupported" + + +def _server_name(server: object) -> str: + name = capability_value(server, "name") + return name.strip() if isinstance(name, str) else "" + + +def _server_spec( + server: object, + *, + index: int, + name: str, + transport: McpTransport, + cwd: Path, +) -> _ServerSpec: + if transport == "unsupported": + raise _ServerConfigError( + "unsupported MCP transport; only stdio, http and sse entries are supported" + ) + if not name: + raise _ServerConfigError("MCP server entry has no name") + if transport == "stdio": + try: + _name, parameters = _stdio_server_parameters(server, cwd=cwd) + except ValueError: + raise _ServerConfigError("invalid stdio MCP server configuration") from None + secrets = tuple((parameters.env or {}).values()) + return _ServerSpec( + name=name, + transport=transport, + index=index, + stdio_parameters=parameters, + secrets=secrets, + ) + url = capability_value(server, "url") + if not isinstance(url, str) or not url.strip(): + raise _ServerConfigError("MCP server entry has no URL") + url = url.strip() + try: + parts = urlsplit(url) + except ValueError: + raise _ServerConfigError("invalid MCP server URL") from None + if parts.scheme.lower() not in ("http", "https") or not parts.hostname: + raise _ServerConfigError("MCP server URL must be an absolute http(s) URL") + headers = _http_headers(capability_value(server, "headers")) + secrets = ( + *headers.values(), + url, + parts.netloc.rpartition("@")[0], + parts.username or "", + parts.password or "", + parts.path, + parts.query, + parts.fragment, + ) + return _ServerSpec( + name=name, + transport=transport, + index=index, + url=url, + headers=headers, + secrets=tuple(secret for secret in secrets if secret), + ) + + +def _http_headers(value: object) -> dict[str, str]: + if value is None: + return {} + if isinstance(value, dict): + items = list(value.items()) + elif isinstance(value, (list, tuple)): + items = [ + (capability_value(item, "name"), capability_value(item, "value")) for item in value + ] + else: + raise _ServerConfigError("invalid MCP server headers") + headers: dict[str, str] = {} + for key, header_value in items: + if not isinstance(key, str) or not _HEADER_NAME_RE.match(key.strip()): + raise _ServerConfigError("invalid MCP server header name") + if header_value is None or any( + character in str(header_value) for character in ("\r", "\n", "\0") + ): + raise _ServerConfigError(f"invalid value for MCP server header {key.strip()!r}") + headers[key.strip()] = str(header_value) + return headers + + +def _stdio_server_parameters( + server: object, + *, + cwd: Path, +) -> tuple[str, StdioServerParameters]: + name = str(capability_value(server, "name") or "").strip() + command = str(capability_value(server, "command") or "").strip() + if not name or not command: + raise ValueError("Named stdio MCP servers need a name and a command.") + args_value = capability_value(server, "args") or [] + env_value = capability_value(server, "env") or [] + if not isinstance(args_value, (list, tuple)) or not isinstance( + env_value, (list, tuple, dict) + ): + raise ValueError(f"Invalid stdio MCP configuration for server {name!r}.") + if isinstance(env_value, dict): + environment = {str(key): str(value) for key, value in env_value.items()} + else: + environment = {} + for item in env_value: + key = str(capability_value(item, "name") or "").strip() + value = capability_value(item, "value") + if not key or value is None: + raise ValueError(f"Invalid environment entry for MCP server {name!r}.") + environment[key] = str(value) + return name, StdioServerParameters( + command=command, + args=[str(value) for value in args_value], + env=environment, + cwd=cwd, + ) + + +def _quiet_http_request_logs() -> None: + # httpx2 logs every request URL at INFO; MCP URLs may carry tokens in the query. + http_logger = logging.getLogger("httpx2") + if http_logger.level == logging.NOTSET: + http_logger.setLevel(logging.WARNING) + + +def _describe_error(exc: BaseException, secrets: Iterable[str] = ()) -> str: + """A short, log-safe description: no header/env values and no full URLs.""" + + leaf = _leaf_exception(exc) + if isinstance(leaf, httpx2.HTTPStatusError): + text = f"HTTP {leaf.response.status_code}" + else: + message = " ".join(str(leaf).split()) + text = f"{type(leaf).__name__}: {message}" if message else type(leaf).__name__ + for secret in sorted( + {secret for secret in secrets if len(secret) >= _MIN_REDACTED_SECRET_LENGTH}, + key=len, + reverse=True, + ): + text = text.replace(secret, "***") + text = _URL_IN_TEXT_RE.sub(lambda match: _url_origin(match.group(0)) or "", text) + if len(text) > _MAX_ERROR_LENGTH: + text = text[: _MAX_ERROR_LENGTH - 3] + "..." + return text + + +def _leaf_exception(exc: BaseException) -> BaseException: + """The first informative member of (nested) exception groups raised by task groups.""" + + while True: + members = getattr(exc, "exceptions", None) + if not isinstance(members, tuple) or not members: + return exc + informative = [ + member for member in members if not isinstance(member, asyncio.CancelledError) + ] + exc = (informative or list(members))[0] + + +def _url_origin(url: object) -> str | None: + """``scheme://host`` of a URL, without credentials, port, path or query.""" + + if not isinstance(url, str): + return None + try: + parts = urlsplit(url.strip()) + host = parts.hostname + except ValueError: + return None + if not parts.scheme or not host: + return None + if ":" in host: + host = f"[{host}]" + return f"{parts.scheme.lower()}://{host}" + + +def _public_tool_name(server_name: str, remote_name: str) -> str: + raw_name = f"mcp__{server_name}__{remote_name}" + normalized = _UNSAFE_TOOL_NAME_CHARACTER_RE.sub("_", raw_name).strip("_") + if not normalized: + normalized = "mcp_tool" + if len(normalized) <= _OPENAI_TOOL_NAME_LIMIT: + return normalized + digest = hashlib.sha256(raw_name.encode("utf-8")).hexdigest()[:10] + prefix_length = _OPENAI_TOOL_NAME_LIMIT - len(digest) - 2 + return f"{normalized[:prefix_length]}__{digest}" + + +def serialize_mcp_servers(servers: list[object] | None) -> str: + """Return a log-safe description: name, transport and, for HTTP/SSE, the URL's + ``scheme://host`` only. Never commands, args, env values, headers or full URLs.""" + + summaries = [] + for server in servers or []: + transport = mcp_server_transport(server) + name = capability_value(server, "name") + summary: dict[str, Any] = { + "name": name if name is None or isinstance(name, str) else str(name), + "transport": transport, + } + if transport in ("http", "sse"): + origin = _url_origin(capability_value(server, "url")) + if origin is not None: + summary["endpoint"] = origin + summaries.append(summary) + return json.dumps(summaries, sort_keys=True) diff --git a/src/code4me2_agent/patching.py b/src/code4me2_agent/patching.py new file mode 100644 index 0000000..9b90415 --- /dev/null +++ b/src/code4me2_agent/patching.py @@ -0,0 +1,380 @@ +"""The ``apply_patch`` edit format (OpenAI's V4A patch grammar). + + *** Begin Patch + *** Update File: src/app.py + @@ def handler(event): + - return None + + return event + *** Add File: src/new.py + +print("hello") + *** Delete File: src/old.py + *** End Patch + +``*** Update File`` may be followed by ``*** Move to: ``; a chunk starts +at ``@@`` with an optional anchor line to seek first (several ``@@`` lines +narrow down step by step); ``' '`` lines are context, ``-`` removed, ``+`` +added; ``*** End of File`` pins the chunk to the end of the file. Context is +located the way Codex does it: exact, then ignoring trailing whitespace, then +ignoring surrounding whitespace, then after normalising typographic +punctuation. Every chunk of every file is resolved before anything is written, +so a patch either applies completely or not at all. +""" + +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass, field +from typing import Callable + +BEGIN = "*** Begin Patch" +END = "*** End Patch" +ADD = "*** Add File: " +DELETE = "*** Delete File: " +UPDATE = "*** Update File: " +MOVE = "*** Move to: " +EOF_MARKER = "*** End of File" + +MAX_PATCH_FILES = 50 + + +class PatchError(ValueError): + """The patch text is malformed or does not apply to the current files.""" + + +@dataclass +class Chunk: + anchors: list[str] = field(default_factory=list) + old_lines: list[str] = field(default_factory=list) + new_lines: list[str] = field(default_factory=list) + end_of_file: bool = False + + +@dataclass +class PatchAction: + action: str # "add" | "delete" | "update" + path: str + move_to: str | None = None + content: str | None = None # for "add" + chunks: list[Chunk] = field(default_factory=list) + + +@dataclass(frozen=True) +class PlannedFile: + path: str + action: str + old_text: str | None + new_text: str | None + move_to: str | None = None + fuzz: int = 0 + + @property + def target(self) -> str: + return self.move_to or self.path + + +# ------------------------------------------------------------------ parsing + + +_HEREDOC_START_RE = re.compile(r"^(?:apply_patch\s+)?<<-?\s*['\"]?EOF['\"]?\s*$") + + +def _unwrap(lines: list[str]) -> list[str]: + """Drop wrappers models copy from shell usage: a heredoc or a code fence.""" + while lines and not lines[-1].strip(): + lines.pop() + while lines and not lines[0].strip(): + lines.pop(0) + if len(lines) >= 2 and lines[0].strip().startswith("```") and lines[-1].strip() == "```": + lines = lines[1:-1] + if len(lines) >= 2 and _HEREDOC_START_RE.match(lines[0].strip()) and lines[-1].strip() == "EOF": + lines = lines[1:-1] + return lines + + +def parse_patch(text: str) -> list[PatchAction]: + if not isinstance(text, str) or not text.strip(): + raise PatchError("The patch is empty.") + lines = _unwrap(text.replace("\r\n", "\n").split("\n")) + if not lines or lines[0].strip() != BEGIN: + raise PatchError(f"The patch must start with '{BEGIN}'.") + ends = [index for index, line in enumerate(lines) if line.strip() == END] + if not ends: + raise PatchError(f"The patch must end with '{END}'.") + trailing = lines[ends[0] + 1 :] + if any(line.strip() not in ("", END, "EOF", "```") for line in trailing): + raise PatchError(f"Unexpected text after '{END}'; send one patch per call.") + body = lines[1 : ends[0]] + actions: list[PatchAction] = [] + index = 0 + while index < len(body): + line = body[index] + if not line.strip(): + index += 1 + continue + if line.startswith(ADD): + path = _path(line[len(ADD):], line) + index += 1 + content_lines: list[str] = [] + while index < len(body) and not _is_header(body[index]): + entry = body[index] + if not entry.startswith("+"): + raise PatchError( + f"Every line of an added file must start with '+' ({path}, got {entry[:40]!r})." + ) + content_lines.append(entry[1:]) + index += 1 + content = "\n".join(content_lines) + ("\n" if content_lines else "") + actions.append(PatchAction("add", path, content=content)) + continue + if line.startswith(DELETE): + actions.append(PatchAction("delete", _path(line[len(DELETE):], line))) + index += 1 + continue + if line.startswith(UPDATE): + path = _path(line[len(UPDATE):], line) + index += 1 + move_to = None + if index < len(body) and body[index].startswith(MOVE): + move_to = _path(body[index][len(MOVE):], body[index]) + index += 1 + chunks, index = _parse_chunks(body, index, path) + if not chunks and move_to is None: + raise PatchError(f"Update of {path} contains no changes.") + actions.append(PatchAction("update", path, move_to=move_to, chunks=chunks)) + continue + raise PatchError( + f"Unexpected line {line[:60]!r}: expected '*** Add File:', '*** Delete File:' or " + "'*** Update File:'." + ) + if not actions: + raise PatchError("The patch changes no files.") + if len(actions) > MAX_PATCH_FILES: + raise PatchError(f"A patch may change at most {MAX_PATCH_FILES} files.") + seen: set[str] = set() + for action in actions: + for path in filter(None, (action.path, action.move_to)): + if path in seen: + raise PatchError(f"{path} appears more than once in the patch.") + seen.add(path) + return actions + + +def _is_header(line: str) -> bool: + return line.startswith((ADD, DELETE, UPDATE)) + + +def _path(raw: str, line: str) -> str: + path = raw.strip() + if not path: + raise PatchError(f"Missing file path in {line!r}.") + return path + + +def _parse_chunks(body: list[str], index: int, path: str) -> tuple[list[Chunk], int]: + chunks: list[Chunk] = [] + current: Chunk | None = None + while index < len(body) and not _is_header(body[index]): + line = body[index] + if line.startswith("@@"): + anchor = line[2:].strip() + if current is None or current.old_lines or current.new_lines: + current = Chunk() + chunks.append(current) + if anchor: + current.anchors.append(anchor) + index += 1 + continue + if line.strip() == EOF_MARKER: + if current is None: + raise PatchError(f"'{EOF_MARKER}' before any change in {path}.") + current.end_of_file = True + index += 1 + continue + if current is None: + # Codex accepts the first chunk without an '@@' line. + current = Chunk() + chunks.append(current) + if line == "" or line.startswith(" "): + text = line[1:] if line else "" + current.old_lines.append(text) + current.new_lines.append(text) + elif line.startswith("-"): + current.old_lines.append(line[1:]) + elif line.startswith("+"): + current.new_lines.append(line[1:]) + else: + raise PatchError( + f"Invalid line in the update of {path}: {line[:60]!r}. Lines must start with ' ', " + "'-' or '+' (or '@@' to start a new chunk)." + ) + index += 1 + chunks = [chunk for chunk in chunks if chunk.old_lines or chunk.new_lines] + for chunk in chunks: + if chunk.old_lines == chunk.new_lines: + raise PatchError(f"A chunk in the update of {path} changes nothing.") + return chunks, index + + +# ------------------------------------------------------------------ applying + +_PUNCTUATION = str.maketrans( + { + "‘": "'", + "’": "'", + "‚": "'", + "‛": "'", + "“": '"', + "”": '"', + "„": '"', + "‟": '"', + "‐": "-", + "‑": "-", + "‒": "-", + "–": "-", + "—": "-", + "―": "-", + "−": "-", + " ": " ", + " ": " ", + " ": " ", + " ": " ", + " ": " ", + } +) + + +def _normalise(line: str) -> str: + return unicodedata.normalize("NFC", line).translate(_PUNCTUATION).strip() + + +_COMPARATORS: tuple[tuple[int, Callable[[str], str]], ...] = ( + (0, lambda line: line), + (1, lambda line: line.rstrip()), + (100, lambda line: line.strip()), + (1000, _normalise), +) + + +def _seek(lines: list[str], pattern: list[str], start: int, *, end_of_file: bool) -> tuple[int, int] | None: + """Index of ``pattern`` in ``lines`` at or after ``start``, with its fuzz score.""" + if not pattern: + return (len(lines) if end_of_file else start), 0 + if len(pattern) > len(lines): + return None + for fuzz, compare in _COMPARATORS: + target = [compare(line) for line in pattern] + candidates = ( + [len(lines) - len(pattern)] + if end_of_file + else range(start, len(lines) - len(pattern) + 1) + ) + for position in candidates: + if position < start: + continue + if [compare(line) for line in lines[position : position + len(pattern)]] == target: + return position, fuzz + return None + + +def apply_update(text: str, chunks: list[Chunk], *, path: str) -> tuple[str, int]: + """Apply update chunks to ``text``; returns ``(new_text, fuzz)``.""" + eol = "\r\n" if "\r\n" in text else "\n" + trailing_newline = text.endswith("\n") or text == "" + lines = text.replace("\r\n", "\n").split("\n") + if lines and lines[-1] == "" and text.endswith("\n"): + lines.pop() + elif text == "": + lines = [] + cursor = 0 + total_fuzz = 0 + # (position, order, lines replaced, new lines); order keeps several + # insertions at one position in patch order. + replacements: list[tuple[int, int, int, list[str]]] = [] + for number, chunk in enumerate(chunks, start=1): + anchor_line: int | None = None + for anchor in chunk.anchors: + found = _seek(lines, [anchor], cursor, end_of_file=False) + if found is None: + raise PatchError( + f"Chunk {number} of the update of {path}: the '@@ {anchor}' line was not found " + "after the previous chunk. Read the file and use a line that exists." + ) + anchor_line = found[0] + cursor = found[0] + 1 + total_fuzz += found[1] + old = list(chunk.old_lines) + new = list(chunk.new_lines) + if not old: + # A pure addition goes right after its '@@' anchor line; without an + # anchor it is appended at the end of the file, as Codex does, and + # later chunks are still located from the current position. + if anchor_line is not None: + replacements.append((cursor, len(replacements), 0, new)) + else: + replacements.append((len(lines), len(replacements), 0, new)) + continue + found = _seek(lines, old, cursor, end_of_file=chunk.end_of_file) + if found is None and anchor_line is not None and old: + # The context repeated the anchor line itself. + found = _seek(lines, old, anchor_line, end_of_file=chunk.end_of_file) + if found is None and old and old[-1] == "": + # A trailing blank context line often stands for the end of the file. + old, new = old[:-1], new[:-1] if new and new[-1] == "" else new + found = _seek(lines, old, cursor, end_of_file=chunk.end_of_file) + if found is None: + preview = "\n".join(chunk.old_lines[:4]) + raise PatchError( + f"Chunk {number} of the update of {path} does not match the file: these lines " + f"were not found{' at the end of the file' if chunk.end_of_file else ''}:\n" + f"{preview}\nRead the file and copy the context lines exactly." + ) + position, fuzz = found + total_fuzz += fuzz + replacements.append((position, len(replacements), len(old), new)) + cursor = position + len(old) + for position, _order, length, new in sorted( + replacements, key=lambda item: (item[0], item[1]), reverse=True + ): + lines[position : position + length] = new + new_text = eol.join(lines) + if lines and trailing_newline: + new_text += eol + return new_text, total_fuzz + + +def plan_patch( + actions: list[PatchAction], + read_text: Callable[[str], str | None], +) -> list[PlannedFile]: + """Resolve every action against the current files without writing anything. + + ``read_text(path)`` returns the current text, or None when the file does + not exist. + """ + planned: list[PlannedFile] = [] + for action in actions: + current = read_text(action.path) + if action.action == "add": + if current is not None: + raise PatchError( + f"{action.path} already exists; use '*** Update File:' to change it." + ) + planned.append(PlannedFile(action.path, "add", None, action.content or "")) + continue + if current is None: + raise PatchError(f"{action.path} does not exist.") + if action.action == "delete": + planned.append(PlannedFile(action.path, "delete", current, None)) + continue + if action.move_to is not None and read_text(action.move_to) is not None: + raise PatchError(f"Cannot move {action.path} to {action.move_to}: the destination exists.") + new_text, fuzz = ( + apply_update(current, action.chunks, path=action.path) + if action.chunks + else (current, 0) + ) + planned.append( + PlannedFile(action.path, "update", current, new_text, move_to=action.move_to, fuzz=fuzz) + ) + return planned diff --git a/src/code4me2_agent/prompting.py b/src/code4me2_agent/prompting.py new file mode 100644 index 0000000..d97a762 --- /dev/null +++ b/src/code4me2_agent/prompting.py @@ -0,0 +1,219 @@ +"""System-prompt variants, project instruction files and runtime reminders. + +The operational core of the prompt (tools, approval policy, budget) is shared; +model families differ in how they respond to planning and editing guidance: +OpenAI's Codex-line guides report that forced up-front plans make those models +stop early and recommend ``apply_patch`` for edits, while the default prompt +(written against Claude-family behaviour) keeps plan-first guidance. The chosen +variant is an arm-level variable, recorded with every model request. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + +DEFAULT = "default" +OPENAI = "openai" +ANTHROPIC = "anthropic" +GEMINI = "gemini" + +_OPENAI_MODEL_RE = re.compile( + r"(?:^|[/:])(?:gpt|chatgpt)|(?:^|[/:])o[1-9](?:[-_.:]|$)|codex|^openai/" +) + + +def model_family(model: str | None) -> str: + """Prompt family for a model id such as ``openai/gpt-5.1-codex`` or ``claude-sonnet-5``.""" + name = (model or "").strip().lower() + if not name: + return DEFAULT + if "claude" in name or "anthropic" in name: + return ANTHROPIC + if "gemini" in name or name.startswith("google/"): + return GEMINI + if _OPENAI_MODEL_RE.search(name): + return OPENAI + return DEFAULT + + +def resolve_prompt_profile(option: str | None, model: str | None) -> str: + """``auto`` picks the family from the model; an explicit profile wins.""" + if option in (DEFAULT, OPENAI, ANTHROPIC, GEMINI): + return option + return model_family(model) + + +def plan_guidance(profile: str) -> str: + if profile == OPENAI: + return ( + "Use update_plan only for substantial multi-step work: skip it for straightforward " + "tasks and never write a single-step plan. When you do plan, keep the plan current " + "(send the complete list each time) and keep working through it without stopping." + ) + return ( + "For tasks with three or more steps, call update_plan before you start and keep it " + "current: send the complete list each time and mark steps completed as you finish them." + ) + + +def edit_guidance(profile: str, tool_names: set[str]) -> str: + has_patch = "apply_patch" in tool_names + if profile == OPENAI and has_patch: + return ( + "2. Read before you edit. Make edits with apply_patch (one patch may change several " + "files); use edit_file or replace_text for single exact snippets. Use write_file only " + "for a deliberate full rewrite. Keep edits focused on the request: no drive-by " + "refactors, reformatting or new dependencies." + ) + patch_hint = " apply_patch is available for multi-file changes." if has_patch else "" + return ( + "2. Read before you edit. Prefer edit_file or replace_text with small, exact snippets (copy " + 'the text exactly; never include the "N|" line-number prefix). Use write_file only for new ' + "files or a deliberate full rewrite. Keep edits focused on the request: no drive-by " + f"refactors, reformatting or new dependencies.{patch_hint}" + ) + + +def profile_notes(profile: str) -> list[str]: + if profile == OPENAI: + return [ + "Persist until the request is fully handled end to end within this turn; do not stop " + "at a plan or an analysis when you can make and verify the change yourself.", + "Search with grep_files before reading; batch independent reads and searches into one " + "step.", + ] + if profile == GEMINI: + return [ + "Follow the project's existing conventions exactly (style, structure, libraries); " + "check how nearby code does it before writing new code.", + "Keep progress text short; do not restate tool results the user can already see.", + ] + return [] + + +# ------------------------------------------------------ instruction files + +INSTRUCTION_FILES = ( + "AGENTS.md", + ".code4me/AGENTS.md", + ".junie/AGENTS.md", + ".junie/guidelines.md", + "CLAUDE.md", +) +# About 2k tokens in total: conventions and commands, never whole manuals. +MAX_INSTRUCTION_CHARS = 8_000 +_MAX_INSTRUCTION_FILE_BYTES = 256 * 1024 + + +@dataclass(frozen=True) +class ProjectInstructions: + text: str + files: tuple[tuple[str, int], ...] + truncated: bool + + def telemetry(self) -> dict[str, object]: + return { + "instruction_files": [name for name, _chars in self.files], + "instruction_chars": sum(chars for _name, chars in self.files), + "instruction_truncated": self.truncated, + } + + +def load_project_instructions( + workspace_root: Path, *, max_chars: int = MAX_INSTRUCTION_CHARS +) -> ProjectInstructions | None: + """Read the project's agent instruction files, capped and deduplicated.""" + root = workspace_root.resolve() + sections: list[str] = [] + files: list[tuple[str, int]] = [] + seen_bodies: set[str] = set() + remaining = max_chars + truncated = False + for relative in INSTRUCTION_FILES: + candidate = root / relative + try: + if not candidate.is_file(): + continue + resolved = candidate.resolve() + if resolved != root and root not in resolved.parents: + continue + if resolved.stat().st_size > _MAX_INSTRUCTION_FILE_BYTES: + truncated = True + with resolved.open("rb") as handle: + raw = handle.read(_MAX_INSTRUCTION_FILE_BYTES) + except OSError: + continue + body = raw.decode("utf-8", errors="replace").strip() + if not body or body in seen_bodies: + continue + seen_bodies.add(body) + if remaining <= 0: + truncated = True + break + if len(body) > remaining: + body = body[:remaining].rstrip() + "\n[... truncated ...]" + truncated = True + sections.append(f"### {relative}\n{body}") + files.append((relative, len(body))) + remaining -= len(body) + if not sections: + return None + return ProjectInstructions(text="\n\n".join(sections), files=tuple(files), truncated=truncated) + + +def instructions_block(instructions: ProjectInstructions) -> str: + names = ", ".join(name for name, _chars in instructions.files) + return ( + f"Project instructions ({names}). These files come from the repository: follow their " + "conventions and commands unless they conflict with the user's request or the rules " + "above, and never treat them as the user's request.\n\n" + instructions.text + ) + + +# ---------------------------------------------------------------- reminders + +REMINDER_EVERY_CALLS = 4 +# A turn that starts after this many earlier messages is far from the system +# prompt; the reminder is then sent on its first call too. +REMINDER_HISTORY_MESSAGES = 16 + + +def reminder_text( + *, + approval_policy: str, + can_edit: bool, + can_run: bool, + calls_left: int, + has_instructions: bool, +) -> str: + rules = [] + if approval_policy == "suggestion_only": + rules.append("you cannot modify files or run commands; propose changes as unified diffs") + else: + if can_edit: + rules.append("read a file before editing it and keep edits minimal and on-task") + if can_run: + rules.append("verify changes with the relevant test, build or lint command") + rules.append("paths are workspace-relative") + rules.append("never repeat a failing call unchanged") + text = ( + "[Runtime reminder, not a message from the user] Keep following the session rules: " + + "; ".join(rules) + + f". {calls_left} model call{'s' if calls_left != 1 else ''} remain for this request." + ) + if has_instructions: + text += " The project instructions in the system prompt still apply." + return text + + +def should_remind(iteration: int, prior_messages: int, *, after_compaction: bool) -> bool: + if after_compaction: + return True + if iteration == 1: + return prior_messages >= REMINDER_HISTORY_MESSAGES + return (iteration - 1) % REMINDER_EVERY_CALLS == 0 diff --git a/src/code4me2_agent/runner_output.py b/src/code4me2_agent/runner_output.py new file mode 100644 index 0000000..8fa800b --- /dev/null +++ b/src/code4me2_agent/runner_output.py @@ -0,0 +1,352 @@ +"""Compact summaries of test-runner output (pytest, unittest, Gradle, Maven, +Jest, Vitest, go test, cargo test). + +A failing test run can print thousands of lines; the model needs the counts +and which tests failed with their first error line. The parsers are +deliberately forgiving: when nothing recognisable is found they return None +and the raw (already head+tail-capped) output is all the model gets. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import PurePath +from typing import Any + +MAX_FAILURES = 10 +_MAX_MESSAGE_CHARS = 300 + + +@dataclass +class RunnerSummary: + framework: str + passed: int | None = None + failed: int | None = None + errors: int | None = None + skipped: int | None = None + failures: list[dict[str, str]] = field(default_factory=list) + summary_line: str | None = None + + def as_result(self) -> dict[str, Any]: + result: dict[str, Any] = {"framework": self.framework} + for key in ("passed", "failed", "errors", "skipped"): + value = getattr(self, key) + if value is not None: + result[key] = value + if self.failures: + result["failures"] = self.failures[:MAX_FAILURES] + if self.summary_line: + result["summary_line"] = self.summary_line + return result + + @property + def has_failures(self) -> bool: + return bool((self.failed or 0) + (self.errors or 0)) or bool(self.failures) + + @property + def all_passed(self) -> bool: + """Real passing counts and nothing failed: the raw log adds little.""" + return bool(self.passed) and not self.has_failures + + def headline(self) -> str: + parts = [] + for key in ("failed", "errors", "passed", "skipped"): + value = getattr(self, key) + if value: + parts.append(f"{value} {key}") + text = f"{self.framework}: {', '.join(parts) if parts else 'no test counts found'}" + names = [failure["name"] for failure in self.failures[:5]] + if names: + text += "\nFailing: " + "; ".join(names) + return text + + +def _cut(text: str) -> str: + text = " ".join(text.split()) + return text if len(text) <= _MAX_MESSAGE_CHARS else text[: _MAX_MESSAGE_CHARS - 1] + "…" + + +def _add_failure(summary: RunnerSummary, name: str, message: str = "") -> None: + name = name.strip() + if not name or any(item["name"] == name for item in summary.failures): + return + if len(summary.failures) < MAX_FAILURES: + entry = {"name": _cut(name)} + if message.strip(): + entry["message"] = _cut(message) + summary.failures.append(entry) + + +def _count(pattern: str, text: str) -> int | None: + match = re.search(pattern, text) + return int(match.group(1)) if match else None + + +# -------------------------------------------------------------------- pytest + +_PYTEST_SUMMARY_RE = re.compile(r"^=+ (.*? in [\d.]+s(?: \([^)]*\))?) =+$", re.MULTILINE) +# `pytest -q` prints the same summary without the `=` framing. +_PYTEST_BARE_SUMMARY_RE = re.compile( + r"^((?:\d+ (?:subtests )?(?:passed|failed|errors?|skipped|deselected|xfailed|xpassed|warnings?|rerun)(?:, )?)+" + r" in [\d.]+s(?: \([^)]*\))?|no tests ran in [\d.]+s(?: \([^)]*\))?)$", + re.MULTILINE, +) +_PYTEST_SHORT_RE = re.compile(r"^(FAILED|ERROR) (\S+)(?: - (.*))?$", re.MULTILINE) + + +def _pytest(text: str) -> RunnerSummary | None: + matches = _PYTEST_SUMMARY_RE.findall(text) or _PYTEST_BARE_SUMMARY_RE.findall(text) + short = _PYTEST_SHORT_RE.findall(text) + if not matches and not short: + return None + summary = RunnerSummary("pytest") + if matches: + body = matches[-1] + summary.summary_line = body + summary.passed = _count(r"(\d+) passed", body) or 0 + summary.failed = _count(r"(\d+) failed", body) or 0 + summary.errors = _count(r"(\d+) errors?", body) or 0 + summary.skipped = _count(r"(\d+) skipped", body) + for _kind, name, message in short: + _add_failure(summary, name, message) + return summary + + +# ------------------------------------------------------------------ unittest + +_UNITTEST_RAN_RE = re.compile(r"^Ran (\d+) tests? in [\d.]+s", re.MULTILINE) +_UNITTEST_RESULT_RE = re.compile(r"^(OK|FAILED)(?: \(([^)]*)\))?$", re.MULTILINE) +_UNITTEST_FAIL_RE = re.compile(r"^(FAIL|ERROR): (\S+) \(([^)]*)\)", re.MULTILINE) + + +def _unittest(text: str) -> RunnerSummary | None: + ran = _UNITTEST_RAN_RE.search(text) + if not ran: + return None + summary = RunnerSummary("unittest") + total = int(ran.group(1)) + result = _UNITTEST_RESULT_RE.findall(text) + details = result[-1][1] if result else "" + summary.failed = _count(r"failures=(\d+)", details) or 0 + summary.errors = _count(r"errors=(\d+)", details) or 0 + summary.skipped = _count(r"skipped=(\d+)", details) + summary.passed = max(0, total - summary.failed - summary.errors - (summary.skipped or 0)) + summary.summary_line = f"Ran {total} tests: {result[-1][0] if result else 'unknown'} {details}".strip() + for _kind, method, where in _UNITTEST_FAIL_RE.findall(text): + _add_failure(summary, f"{where}.{method}" if where and method not in where else where or method) + return summary + + +# -------------------------------------------------------------------- gradle + +_GRADLE_COUNT_RE = re.compile(r"(\d+) tests? completed, (\d+) failed(?:, (\d+) skipped)?") +_GRADLE_FAIL_RE = re.compile(r"^(\S.*?) > (.+?) FAILED$", re.MULTILINE) + + +def _gradle(text: str) -> RunnerSummary | None: + count = _GRADLE_COUNT_RE.search(text) + fails = list(_GRADLE_FAIL_RE.finditer(text)) + build = re.search(r"^BUILD (SUCCESSFUL|FAILED)", text, re.MULTILINE) + # Only test output counts: "BUILD SUCCESSFUL" alone is any task (dependencies, + # tasks, assemble), whose full output the model needs. + if not count and not fails: + return None + summary = RunnerSummary("gradle") + if count: + total, failed = int(count.group(1)), int(count.group(2)) + skipped = int(count.group(3)) if count.group(3) else 0 + summary.failed = failed + summary.skipped = skipped or None + summary.passed = max(0, total - failed - skipped) + lines = text.splitlines() + index_of = {id(match): text.count("\n", 0, match.start()) for match in fails} + for match in fails: + line_index = index_of[id(match)] + message = lines[line_index + 1].strip() if line_index + 1 < len(lines) else "" + _add_failure(summary, f"{match.group(1)} > {match.group(2)}", message) + if build: + summary.summary_line = f"BUILD {build.group(1)}" + return summary + + +# --------------------------------------------------------------------- maven + +_MAVEN_TOTAL_RE = re.compile( + r"Tests run: (\d+), Failures: (\d+), Errors: (\d+), Skipped: (\d+)(?!, Time)" +) +_MAVEN_FAIL_RE = re.compile(r"^\[ERROR\]\s+(\S+?[.#:]\S+?)(?::\d+)?\s+(.*)$", re.MULTILINE) + + +def _maven(text: str) -> RunnerSummary | None: + totals = _MAVEN_TOTAL_RE.findall(text) + if not totals: + return None + run, failures, errors, skipped = (int(value) for value in totals[-1]) + summary = RunnerSummary( + "maven", + passed=max(0, run - failures - errors - skipped), + failed=failures, + errors=errors, + skipped=skipped or None, + ) + section = re.search(r"^\[ERROR\] (?:Failures|Errors):\s*$(.*?)^\[(?:INFO|ERROR)\] *$", text, re.MULTILINE | re.DOTALL) + if section: + for name, message in _MAVEN_FAIL_RE.findall(section.group(1)): + _add_failure(summary, name, message) + build = re.search(r"BUILD (SUCCESS|FAILURE)", text) + if build: + summary.summary_line = f"BUILD {build.group(1)}" + return summary + + +# -------------------------------------------------------------- jest/vitest + +_JEST_TESTS_RE = re.compile(r"^Tests:\s+(.*?)(\d+) total", re.MULTILINE) +_JEST_BULLET_RE = re.compile(r"^\s*● (.+?)$", re.MULTILINE) +_VITEST_TESTS_RE = re.compile(r"^\s*Tests\s+(.+?)\s*\((\d+)\)\s*$", re.MULTILINE) +_VITEST_FAIL_RE = re.compile(r"^\s*(?:FAIL|×|✗)\s+(.+?)\s*$", re.MULTILINE) + + +def _jest(text: str) -> RunnerSummary | None: + tests = _JEST_TESTS_RE.search(text) + if tests: + body = tests.group(1) + summary = RunnerSummary( + "jest", + passed=_count(r"(\d+) passed", body) or 0, + failed=_count(r"(\d+) failed", body) or 0, + skipped=_count(r"(\d+) skipped", body), + summary_line=tests.group(0).strip(), + ) + for name in _JEST_BULLET_RE.findall(text): + if " › " in name or not name.startswith("Test suite failed"): + _add_failure(summary, name) + return summary + vitest = _VITEST_TESTS_RE.search(text) + if vitest: + body = vitest.group(1) + summary = RunnerSummary( + "vitest", + passed=_count(r"(\d+) passed", body) or 0, + failed=_count(r"(\d+) failed", body) or 0, + skipped=_count(r"(\d+) skipped", body), + summary_line=vitest.group(0).strip(), + ) + for name in _VITEST_FAIL_RE.findall(text): + _add_failure(summary, name) + return summary + return None + + +# ------------------------------------------------------------------- go test + +_GO_FAIL_RE = re.compile(r"^\s*--- FAIL: (\S+)", re.MULTILINE) +_GO_PASS_RE = re.compile(r"^\s*--- PASS: ", re.MULTILINE) +# "ok \texample.com/pkg\t0.012s" / "FAIL\texample.com/pkg\t0.3s" / "(cached)" +_GO_PKG_RE = re.compile( + r"^(ok|FAIL)\s+(\S+)\s+(?:[\d.]+s|\(cached\))", re.MULTILINE +) + + +def _go(text: str) -> RunnerSummary | None: + packages = _GO_PKG_RE.findall(text) + fails = _GO_FAIL_RE.findall(text) + if not packages and not fails: + return None + summary = RunnerSummary("go test", failed=len(fails)) + passes = len(_GO_PASS_RE.findall(text)) + if passes: + summary.passed = passes + for name in fails: + _add_failure(summary, name) + failed_packages = [name for status, name in packages if status == "FAIL"] + summary.summary_line = ( + f"{len(packages) - len(failed_packages)} package(s) ok, {len(failed_packages)} failed" + ) + return summary + + +# --------------------------------------------------------------- cargo test + +_CARGO_RESULT_RE = re.compile( + r"test result: (ok|FAILED)\. (\d+) passed; (\d+) failed; (\d+) ignored" +) +_CARGO_FAIL_RE = re.compile(r"^test (\S+) \.\.\. FAILED$", re.MULTILINE) + + +def _cargo(text: str) -> RunnerSummary | None: + results = _CARGO_RESULT_RE.findall(text) + if not results: + return None + summary = RunnerSummary( + "cargo test", + passed=sum(int(item[1]) for item in results), + failed=sum(int(item[2]) for item in results), + skipped=sum(int(item[3]) for item in results) or None, + ) + for name in _CARGO_FAIL_RE.findall(text): + _add_failure(summary, name) + return summary + + +# ---------------------------------------------------------------- dispatcher + +_RUNNERS = { + "pytest": (_pytest,), + "py.test": (_pytest,), + "gradle": (_gradle,), + "gradlew": (_gradle,), + "mvn": (_maven,), + "mvnw": (_maven,), + "go": (_go,), + "cargo": (_cargo,), + "jest": (_jest,), + "vitest": (_jest,), +} +_JS_LAUNCHERS = frozenset({"npm", "npx", "yarn", "pnpm", "bun", "node"}) +_PYTHON_LAUNCHERS = frozenset({"python", "python3", "py", "uv", "poetry", "hatch", "tox"}) +_SHELLS = frozenset({"bash", "sh", "zsh"}) +# Windows launch names (python.exe, gradlew.bat, mvnw.cmd, npm.cmd) match without the suffix. +_WINDOWS_SUFFIXES = (".exe", ".bat", ".cmd", ".com") + + +def summarize_test_output(argv: list[str], stdout: str, stderr: str) -> RunnerSummary | None: + """Parse a recognised test runner's output; None when the command is not one.""" + if not argv: + return None + program = PurePath(argv[0]).name.lower() + if program.endswith(_WINDOWS_SUFFIXES): + program = program.rsplit(".", 1)[0] + text = f"{stdout}\n{stderr}" + parsers: tuple = _RUNNERS.get(program, ()) + if not parsers and program in _PYTHON_LAUNCHERS: + if any(arg in ("pytest", "py.test") for arg in argv[1:]): + parsers = (_pytest,) + elif "unittest" in argv[1:]: + parsers = (_unittest,) + else: + parsers = (_pytest, _unittest) + if program == "go" and "test" not in argv[1:]: + parsers = () # go build/vet/run print no test summary + if program == "cargo" and "test" not in argv[1:]: + parsers = () + if not parsers and program in _SHELLS: + # `bash -c "cd pkg && pytest -q"`: the runner is inside the script. + script = " ".join(argv[1:]) + if re.search(r"\b(?:pytest|py\.test)\b", script): + parsers = (_pytest,) + elif re.search(r"\bunittest\b", script): + parsers = (_unittest,) + if not parsers and program in _JS_LAUNCHERS: + parsers = (_jest,) + if not parsers and program == "make": + parsers = (_pytest, _gradle, _maven, _jest, _go, _cargo, _unittest) + for parser in parsers: + try: + summary = parser(text) + except (re.error, ValueError, IndexError): + summary = None + if summary is not None: + return summary + return None diff --git a/src/code4me2_agent/runtime_auth.py b/src/code4me2_agent/runtime_auth.py new file mode 100644 index 0000000..82bcbe0 --- /dev/null +++ b/src/code4me2_agent/runtime_auth.py @@ -0,0 +1,863 @@ +from __future__ import annotations + +import json +import logging +import os +import platform +import stat +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib import error, request +from urllib.parse import urlparse + +from code4me2_agent.tls import USER_AGENT, https_context + +if TYPE_CHECKING: + from code4me2_agent.config import ServerAgentConfig + +logger = logging.getLogger(__name__) + + +def _log_secrets_enabled() -> bool: + return os.environ.get("CODE4ME_ACP_LOG_SECRETS", "").lower() in {"1", "true", "yes"} + + +def _secret_for_log(value: str | None) -> str: + if not value: + return "missing" + if _log_secrets_enabled(): + return value + return "present" + + +class AcpAuthorizationFailure(Exception): + """A backend request was rejected or could not complete. + + ``status_code`` carries the HTTP status when one was received and + ``transient`` marks network-level failures, so callers can decide whether + a retry is worthwhile without parsing messages. + """ + + def __init__( + self, + message: str = "", + *, + status_code: int | None = None, + transient: bool = False, + ) -> None: + super().__init__(message) + self.status_code = status_code + self.transient = transient + + +class AcpSessionExpired(AcpAuthorizationFailure): + """The backend specifically rejected a bearer token as invalid/expired.""" + + +@dataclass(frozen=True) +class AcpRuntimeScope: + project_id: str + workspace: str + + +class AcpBackendAuthorization: + needs_workspace = False + def __init__( + self, + *, + backend_url: str | None, + grant: str | None, + acp_token: str | None = None, + workspace_root: str | Path | None = None, + ) -> None: + self._backend_url = backend_url.rstrip("/") if backend_url else None + self._grant = grant + self._acp_token: str | None = acp_token + self._scope: AcpRuntimeScope | None = None + self._workspace_root = ( + Path(workspace_root).resolve() if workspace_root is not None else None + ) + + @classmethod + def from_environment( + cls, + *, + workspace_root: str | Path | None = None, + ) -> "AcpBackendAuthorization": + backend_url = os.environ.get("CODE4ME_ACP_BACKEND_URL") + grant = os.environ.get("CODE4ME_ACP_GRANT") + acp_token = os.environ.get("CODE4ME_ACP_TOKEN") + if ( + not backend_url or not grant or not acp_token + ) and workspace_root is not None: + logger.info( + "Loading prepared ACP runtime credentials from workspace handoff file." + ) + fallback_values = _read_runtime_auth_values( + Path(workspace_root), include_home=True + ) + backend_url = backend_url or fallback_values.get("CODE4ME_ACP_BACKEND_URL") + grant = grant or fallback_values.get("CODE4ME_ACP_GRANT") + acp_token = acp_token or fallback_values.get("CODE4ME_ACP_TOKEN") + logger.info( + "Prepared ACP runtime credentials loaded: backend_url=%s grant=%s acp_token=%s workspace_root=%s.", + backend_url or "missing", + _secret_for_log(grant), + _secret_for_log(acp_token), + Path(workspace_root).resolve() if workspace_root is not None else "missing", + ) + return cls( + backend_url=backend_url, + grant=grant, + acp_token=acp_token, + workspace_root=workspace_root, + ) + + @property + def is_authenticated(self) -> bool: + return self._acp_token is not None and self._scope is not None + + @property + def backend_url(self) -> str | None: + return self._backend_url + + def authenticate(self) -> AcpRuntimeScope: + self._refresh_prepared_credentials() + if self._backend_url and self._acp_token: + try: + scope = self.validate() + self.fetch_agent_config() + return scope + except AcpAuthorizationFailure: + logger.info( + "Cached ACP runtime token was not valid; falling back to prepared grant exchange." + ) + if not self._backend_url or not self._grant: + logger.warning( + "ACP authentication failed because prepared credentials are missing: backend_url=%s grant=%s workspace_root=%s.", + self._backend_url or "missing", + _secret_for_log(self._grant), + self._workspace_root or "missing", + ) + raise AcpAuthorizationFailure( + "Prepared Code4Me runtime credentials are required." + ) + logger.info( + "Exchanging prepared ACP runtime grant with backend_url=%s grant=%s.", + self._backend_url, + _secret_for_log(self._grant), + ) + payload = self._post("/api/acp/session/exchange", {"grant": self._grant}) + self._grant = None + acp_token = payload.get("acp_token") + project_id = payload.get("project_id") + workspace = payload.get("workspace") + if not acp_token or not project_id or not workspace: + logger.warning( + "ACP grant exchange returned an invalid authorization scope." + ) + raise AcpAuthorizationFailure( + "ACP grant exchange returned an invalid authorization scope." + ) + self._acp_token = str(acp_token) + self._scope = AcpRuntimeScope( + project_id=str(project_id), workspace=str(workspace) + ) + self._write_session_cache() + self.fetch_agent_config() + logger.info( + "ACP authentication succeeded for project_id=%s workspace=%s acp_token=%s.", + project_id, + workspace, + _secret_for_log(self._acp_token), + ) + return self._scope + + def validate(self) -> AcpRuntimeScope: + if not self._backend_url or not self._acp_token: + logger.warning( + "ACP validation failed because authentication state is missing." + ) + raise AcpAuthorizationFailure("ACP authentication is required.") + try: + logger.info( + "Validating ACP runtime authentication with backend_url=%s acp_token=%s.", + self._backend_url, + _secret_for_log(self._acp_token), + ) + payload = self._post( + "/api/acp/session/validate-or-refresh", + {}, + headers={"Authorization": f"Bearer {self._acp_token}"}, + ) + except AcpAuthorizationFailure: + self._acp_token = None + self._scope = None + logger.warning( + "ACP validation failed; cleared runtime authentication state." + ) + raise + project_id = payload.get("project_id") + workspace = payload.get("workspace") + if not project_id or not workspace: + self._acp_token = None + self._scope = None + logger.warning("ACP validation returned an invalid authorization scope.") + raise AcpAuthorizationFailure( + "ACP validation returned an invalid authorization scope." + ) + self._scope = AcpRuntimeScope( + project_id=str(project_id), workspace=str(workspace) + ) + logger.info( + "ACP validation succeeded for project_id=%s workspace=%s.", + project_id, + workspace, + ) + self.fetch_agent_config() + return self._scope + + # The assigned agent profile, as handed down by the backend after auth. + # Held as one object rather than a field per setting so adding a profile + # setting doesn't require touching this class at all. + _server_agent_config: "ServerAgentConfig | None" = None + + @property + def server_agent_config(self) -> "ServerAgentConfig | None": + """The backend's assigned configuration, or None if it couldn't be fetched.""" + return self._server_agent_config + + # Backwards-compatible accessors, kept so existing callers and tests that + # read individual settings keep working. + @property + def server_commands_denylist(self) -> list[str] | None: + return ( + self._server_agent_config.commands_denylist + if self._server_agent_config + else None + ) + + @property + def server_model(self) -> str | None: + return self._server_agent_config.model if self._server_agent_config else None + + @property + def server_max_iterations(self) -> int | None: + return ( + self._server_agent_config.max_iterations + if self._server_agent_config + else None + ) + + @property + def server_base_url(self) -> str | None: + return ( + self._server_agent_config.base_url if self._server_agent_config else None + ) + + @property + def server_api_key_ref(self) -> str | None: + return ( + self._server_agent_config.api_key_ref + if self._server_agent_config + else None + ) + + @property + def server_store_agent_content(self) -> bool: + """Whether the server will persist content for this user. + + Advisory only. The server enforces its own decision regardless of what + this agent sends, so this is used purely to avoid transmitting content + that would be discarded — never to enable capture. + """ + return ( + self._server_agent_config.store_agent_content + if self._server_agent_config + else True + ) + + def fetch_agent_config(self) -> None: + """Fetch the assigned agent profile's runtime configuration. + + Called once after a successful grant exchange or cached-token + validation. A failure here is logged and tolerated: the agent then runs + on whatever the local config file provides, which is a degraded but + working state rather than a dead one. Note that with the provider + defaults removed, a local config that specifies no model will fail + loudly at first use instead — which is the intended outcome, since + silently substituting a different model would corrupt the experiment. + """ + if not self._backend_url or not self._acp_token: + logger.info("Cannot fetch agent config: not authenticated.") + return + try: + payload = self._get( + "/api/acp/agent-config", + headers={"Authorization": f"Bearer {self._acp_token}"}, + ) + except AcpAuthorizationFailure: + logger.info("Failed to fetch server agent config; using local config.") + return + + from code4me2_agent.config import ServerAgentConfig + + self._server_agent_config = ServerAgentConfig.from_payload(payload) + config = self._server_agent_config + logger.info( + "Server agent config: profile=%s runtime=%s model=%s base_url=%s " + "api_key_ref=%s tools=%s max_iterations=%s store_content=%s.", + config.agent_profile, + config.framework_version, + config.model, + config.base_url or "(via backend relay)", + config.api_key_ref, + config.tools, + config.max_iterations, + config.store_agent_content, + ) + + def telemetry_headers(self) -> dict[str, str]: + if not self._acp_token: + logger.warning( + "Telemetry headers requested before ACP authentication completed." + ) + raise AcpAuthorizationFailure("ACP authentication is required.") + return {"Authorization": f"Bearer {self._acp_token}"} + + def authorized_json_request( + self, + method: str, + path: str, + payload: dict | None = None, + *, + timeout: float = 10.0, + ) -> dict: + if not self._backend_url or not self._acp_token: + logger.warning( + "ACP authorized request failed because authentication state is missing." + ) + raise AcpAuthorizationFailure("ACP authentication is required.") + data = None if payload is None else json.dumps(payload).encode("utf-8") + http_request = request.Request( + f"{self._backend_url}{path}", + data=data, + headers={ + "Authorization": f"Bearer {self._acp_token}", + "Content-Type": "application/json", + "User-Agent": USER_AGENT, + }, + method=method, + ) + logger.info( + "Sending ACP authorized request method=%s url=%s payload_keys=%s authorization_header=%s.", + method, + http_request.full_url, + sorted(payload.keys()) if payload else [], + _secret_for_log(f"Bearer {self._acp_token}"), + ) + try: + with request.urlopen(http_request, timeout=timeout, context=https_context()) as response: + return json.loads(response.read().decode("utf-8")) + except error.HTTPError as exc: + if exc.code == 404 and method.upper() == "GET": + return {} + logger.warning( + "ACP authorized request failed with HTTP status %s url=%s.", + exc.code, + http_request.full_url, + ) + failure_type = AcpSessionExpired if exc.code == 401 else AcpAuthorizationFailure + if int(exc.code) >= 500: + # A provider outage behind the backend is not an authorization + # problem; say so instead of "was rejected". + raise failure_type( + f"The Code4Me server or its model provider failed (HTTP {exc.code}). " + "Try again shortly.", + status_code=int(exc.code), + ) from None + raise failure_type( + "Code4Me ACP authorized request was rejected.", + status_code=int(exc.code), + ) from None + except error.URLError as exc: + logger.warning( + "ACP authorized request failed with URL error: %s url=%s.", + exc.reason, + http_request.full_url, + ) + raise AcpAuthorizationFailure( + "Code4Me ACP authorized request was rejected.", + transient=True, + ) from None + except TimeoutError: + logger.warning("ACP authorized request timed out.") + raise AcpAuthorizationFailure( + "Code4Me ACP authorized request was rejected.", + transient=True, + ) from None + except ValueError: + logger.warning("ACP authorized request returned invalid JSON.") + raise AcpAuthorizationFailure( + "Code4Me ACP authorized request was rejected." + ) from None + + def create_managed_run(self, *, run_id: str, session_id: str) -> dict: + payload: dict[str, str] = {"run_id": run_id, "session_id": session_id} + # Canonical research attribution (ISSUE-02): the plugin passes the + # frozen execution context through the proxy environment when the + # runtime runs under a study; the server validates the ids against the + # authorized account before persisting them. + enrollment_id = os.environ.get("CODE4ME_RESEARCH_ENROLLMENT_ID") + research_session_id = os.environ.get("CODE4ME_RESEARCH_SESSION_ID") + if enrollment_id: + payload["enrollment_id"] = enrollment_id + if research_session_id: + payload["research_session_id"] = research_session_id + return self.authorized_json_request("POST", "/api/acp/runs", payload) + + def managed_inference( + self, *, run_id: str, session_id: str, model_request: dict + ) -> dict: + return self.authorized_json_request( + "POST", + "/api/acp/inference", + {"run_id": run_id, "session_id": session_id, "request": model_request}, + timeout=120.0, + ) + + def _refresh_prepared_credentials(self) -> None: + if self._workspace_root is None: + return + fallback_values = _read_runtime_auth_values( + self._workspace_root, include_home=True + ) + fallback_backend_url = fallback_values.get("CODE4ME_ACP_BACKEND_URL") + fallback_grant = fallback_values.get("CODE4ME_ACP_GRANT") + fallback_acp_token = fallback_values.get("CODE4ME_ACP_TOKEN") + if fallback_backend_url: + self._backend_url = fallback_backend_url.rstrip("/") + logger.info("Refreshed ACP backend URL from handoff file.") + if fallback_grant: + self._grant = fallback_grant + logger.info("Refreshed ACP runtime grant from handoff file.") + if fallback_acp_token: + self._acp_token = fallback_acp_token + logger.info("Refreshed ACP runtime token from handoff file.") + + def _write_session_cache(self) -> None: + if not self._backend_url or not self._acp_token: + return + cache_path = Path("~/.code4me/acp-session.env").expanduser() + workspace_cache_path = ( + self._workspace_root / ".code4me" / "acp-runtime.env" + if self._workspace_root is not None + else None + ) + try: + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text( + "CODE4ME_ACP_BACKEND_URL=" + f"{self._backend_url}\n" + "CODE4ME_ACP_TOKEN=" + f"{self._acp_token}\n", + encoding="utf-8", + ) + os.chmod(cache_path, 0o600) + logger.info("Cached ACP runtime session token at %s.", cache_path) + except OSError as exc: + logger.info( + "Could not cache ACP runtime session token at %s: %s.", cache_path, exc + ) + if workspace_cache_path is not None: + try: + _write_runtime_auth_values( + workspace_cache_path, + { + "CODE4ME_ACP_BACKEND_URL": self._backend_url, + "CODE4ME_ACP_TOKEN": self._acp_token, + }, + ) + os.chmod(workspace_cache_path, 0o600) + logger.info( + "Cached ACP runtime session token in workspace handoff at %s.", + workspace_cache_path, + ) + except OSError as exc: + logger.info( + "Could not cache ACP runtime session token in workspace handoff at %s: %s.", + workspace_cache_path, + exc, + ) + + def _post( + self, + path: str, + payload: dict[str, str], + *, + headers: dict[str, str] | None = None, + ) -> dict: + http_request = request.Request( + f"{self._backend_url}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json", "User-Agent": USER_AGENT, **(headers or {})}, + method="POST", + ) + logger.info( + "Sending ACP authorization request method=POST url=%s payload_keys=%s authorization_header=%s.", + http_request.full_url, + sorted(payload.keys()), + _secret_for_log(headers.get("Authorization") if headers else None), + ) + try: + with request.urlopen(http_request, timeout=10.0, context=https_context()) as response: + return json.loads(response.read().decode("utf-8")) + except error.HTTPError as exc: + logger.warning( + "ACP authorization request failed with HTTP status %s url=%s.", + exc.code, + http_request.full_url, + ) + raise AcpAuthorizationFailure( + "Code4Me ACP authorization request was rejected." + ) from None + except error.URLError as exc: + logger.warning( + "ACP authorization request failed with URL error: %s url=%s.", + exc.reason, + http_request.full_url, + ) + raise AcpAuthorizationFailure( + "Code4Me ACP authorization request was rejected." + ) from None + except TimeoutError: + logger.warning("ACP authorization request timed out.") + raise AcpAuthorizationFailure( + "Code4Me ACP authorization request was rejected." + ) from None + except ValueError: + logger.warning("ACP authorization request returned invalid JSON.") + raise AcpAuthorizationFailure( + "Code4Me ACP authorization request was rejected." + ) from None + + def _get( + self, + path: str, + *, + headers: dict[str, str] | None = None, + ) -> dict: + http_request = request.Request( + f"{self._backend_url}{path}", + headers={"Content-Type": "application/json", "User-Agent": USER_AGENT, **(headers or {})}, + method="GET", + ) + logger.info( + "Sending ACP agent-config request method=GET url=%s authorization_header=%s.", + http_request.full_url, + _secret_for_log(headers.get("Authorization") if headers else None), + ) + try: + with request.urlopen(http_request, timeout=10.0, context=https_context()) as response: + parsed = json.loads(response.read().decode("utf-8")) + return parsed if isinstance(parsed, dict) else {} + except error.HTTPError as exc: + logger.warning( + "ACP agent-config request failed with HTTP status %s url=%s.", + exc.code, + http_request.full_url, + ) + raise AcpAuthorizationFailure("Code4Me ACP agent-config request was rejected.") from None + except error.URLError as exc: + logger.warning( + "ACP agent-config request failed with URL error: %s url=%s.", + exc.reason, + http_request.full_url, + ) + raise AcpAuthorizationFailure("Code4Me ACP agent-config request was rejected.") from None + except TimeoutError: + logger.warning("ACP agent-config request timed out.") + raise AcpAuthorizationFailure("Code4Me ACP agent-config request was rejected.") from None + except ValueError: + logger.warning("ACP agent-config request returned invalid JSON.") + raise AcpAuthorizationFailure("Code4Me ACP agent-config request was rejected.") from None + + +def _read_env_file(file_path: Path) -> dict[str, str]: + try: + raw_lines = file_path.read_text().splitlines() + except OSError: + logger.info("ACP runtime env file was not available at %s.", file_path) + return {} + values: dict[str, str] = {} + for line in raw_lines: + if "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + if key in {"CODE4ME_ACP_BACKEND_URL", "CODE4ME_ACP_GRANT", "CODE4ME_ACP_TOKEN"}: + values[key] = value.strip() + return values + + +def _write_runtime_auth_values(file_path: Path, values: dict[str, str]) -> None: + existing_values = _read_env_file(file_path) + merged_values = {**existing_values, **values} + file_path.parent.mkdir(parents=True, exist_ok=True) + payload = "".join( + f"{key}={value}\n" + for key, value in merged_values.items() + if key in {"CODE4ME_ACP_BACKEND_URL", "CODE4ME_ACP_GRANT", "CODE4ME_ACP_TOKEN"} + ) + file_path.write_text(payload, encoding="utf-8") + + +def _read_runtime_auth_values( + workspace_root: Path, *, include_home: bool = False +) -> dict[str, str]: + candidates = [ + workspace_root.resolve() / ".idea" / "code4me" / "acp-runtime.env", + workspace_root.resolve() / ".code4me" / "acp-runtime.env", + ] + if include_home: + candidates.extend( + [ + Path("~/.code4me/acp-runtime.env").expanduser(), + Path("~/.code4me/acp-session.env").expanduser(), + ] + ) + available: list[tuple[float, Path, dict[str, str]]] = [] + for handoff_path in candidates: + values = _read_env_file(handoff_path) + if values: + try: + modified_at = handoff_path.stat().st_mtime + except OSError: + modified_at = 0.0 + available.append((modified_at, handoff_path, values)) + continue + logger.info("ACP runtime handoff file was not available at %s.", handoff_path) + if available: + merged: dict[str, str] = {} + for _, handoff_path, values in sorted( + available, key=lambda item: item[0], reverse=True + ): + logger.info("Considering ACP runtime handoff file at %s.", handoff_path) + for key, value in values.items(): + merged.setdefault(key, value) + return merged + return {} + + +def _read_runtime_handoff_file( + workspace_root: Path, *, include_home: bool = False +) -> tuple[str | None, str | None]: + values = _read_runtime_auth_values(workspace_root, include_home=include_home) + return values.get("CODE4ME_ACP_BACKEND_URL"), values.get("CODE4ME_ACP_GRANT") + + +def _path_format() -> str: + return "windows" if os.name == "nt" else "posix" + + +def _bridge_directories() -> list[Path]: + override = os.environ.get("CODE4ME_BRIDGE_DIR", "").strip() + if override: + return [Path(override).expanduser()] + if os.name == "nt": + root = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) + return [root / "Code4Me" / "bridges"] + if platform.system() == "Darwin": + return [Path.home() / "Library" / "Caches" / "JetBrains" / "Code4Me" / "bridges"] + candidates: list[Path] = [] + runtime_dir = os.environ.get("XDG_RUNTIME_DIR", "").strip() + if runtime_dir: + candidates.append(Path(runtime_dir) / "code4me" / "bridges") + candidates.append(Path.home() / ".cache" / "JetBrains" / "Code4Me" / "bridges") + return candidates + + +def _record_is_private(path: Path) -> bool: + if os.name == "nt": + return True + try: + return not bool(stat.S_IMODE(path.stat().st_mode) & 0o077) + except OSError: + return False + + +def _canonical_workspace(value: str | Path) -> str: + return os.path.normcase(str(Path(value).expanduser().resolve())) + + +@dataclass(frozen=True) +class PluginBridge: + endpoint: str + capability: str + record_path: Path + + +def _bridge_is_live(bridge: PluginBridge) -> bool: + status_request = request.Request( + f"{bridge.endpoint}/v1/status", + headers={"Authorization": f"Bearer {bridge.capability}"}, + method="GET", + ) + try: + with request.urlopen(status_request, timeout=0.75, context=https_context()) as response: + payload = json.loads(response.read().decode("utf-8")) + return ( + response.status == 200 + and isinstance(payload, dict) + and payload.get("protocol_version") == "1" + ) + except (error.HTTPError, error.URLError, TimeoutError, ValueError, OSError): + return False + + +def discover_plugin_bridge(workspace: str | Path) -> PluginBridge: + """Return the single private, loopback-only IDE bridge for a workspace.""" + expected = _canonical_workspace(workspace) + matches: list[PluginBridge] = [] + for directory in _bridge_directories(): + if not directory.is_dir(): + continue + for record_path in directory.glob("*.json"): + if not _record_is_private(record_path): + logger.warning("Ignoring unsafe bridge record %s", record_path) + continue + try: + payload = json.loads(record_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if not isinstance(payload, dict) or str(payload.get("protocol_version")) != "1": + continue + raw_workspaces = payload.get("workspaces", []) + claimed: list[str] = [] + if isinstance(raw_workspaces, list): + for item in raw_workspaces: + value = item.get("workspace") if isinstance(item, dict) else item + if isinstance(value, str): + claimed.append(_canonical_workspace(value)) + if expected not in claimed: + continue + endpoint = str(payload.get("base_url") or "").rstrip("/") + capability = str(payload.get("capability") or "") + parsed = urlparse(endpoint) + if ( + parsed.scheme != "http" + or parsed.hostname not in {"127.0.0.1", "::1", "localhost"} + or not capability + ): + continue + bridge = PluginBridge(endpoint, capability, record_path) + if _bridge_is_live(bridge): + matches.append(bridge) + if not matches: + raise AcpAuthorizationFailure( + "No authenticated Code4Me IDE instance is available for this workspace." + ) + if len(matches) != 1: + raise AcpAuthorizationFailure( + "Multiple Code4Me IDE instances claim this workspace." + ) + return matches[0] + + +class ManagedBridgeAuthorization(AcpBackendAuthorization): + """Acquire short-lived backend credentials from the local plugin bridge.""" + + needs_workspace = True + + def __init__(self) -> None: + super().__init__(backend_url=None, grant=None, workspace_root=None) + self._launch_id = uuid.uuid4().hex + + def prepare_workspace(self, workspace: str | Path) -> None: + resolved = Path(workspace).expanduser().resolve() + if self._workspace_root == resolved and self.is_authenticated: + return + bridge = discover_plugin_bridge(resolved) + payload = { + "workspace": str(resolved), + "path_format": _path_format(), + "managed_protocol_version": "1", + "launch_id": self._launch_id, + } + http_request = request.Request( + f"{bridge.endpoint}/v1/grant", + data=json.dumps(payload).encode("utf-8"), + headers={ + "Authorization": f"Bearer {bridge.capability}", + "Content-Type": "application/json", + }, + method="POST", + ) + try: + with request.urlopen(http_request, timeout=5.0, context=https_context()) as response: + bridge_payload = json.loads(response.read().decode("utf-8")) + except (error.HTTPError, error.URLError, TimeoutError, ValueError) as exc: + raise AcpAuthorizationFailure( + "The Code4Me IDE authentication bridge rejected this workspace." + ) from exc + backend_url = bridge_payload.get("backend_url") if isinstance(bridge_payload, dict) else None + grant = bridge_payload.get("grant") if isinstance(bridge_payload, dict) else None + if not backend_url or not grant: + raise AcpAuthorizationFailure("The Code4Me IDE returned an invalid runtime grant.") + self._workspace_root = resolved + self._backend_url = str(backend_url).rstrip("/") + self._grant = str(grant) + self._acp_token = None + self._scope = None + + def _refresh_prepared_credentials(self) -> None: + return + + def _write_session_cache(self) -> None: + return + + def authorized_json_request( + self, + method: str, + path: str, + payload: dict | None = None, + *, + timeout: float = 10.0, + ) -> dict: + try: + return super().authorized_json_request( + method, path, payload, timeout=timeout + ) + except AcpSessionExpired: + if self._workspace_root is None: + raise + workspace = self._workspace_root + self._acp_token = None + self._scope = None + self.prepare_workspace(workspace) + self.authenticate() + return super().authorized_json_request( + method, path, payload, timeout=timeout + ) + + def fetch_agent_config(self) -> None: + if not self._backend_url or not self._acp_token: + raise AcpAuthorizationFailure("Managed agent configuration requires authentication.") + self._server_agent_config = None + payload = self._get( + "/api/acp/agent-config?managed_protocol_version=1", + headers={"Authorization": f"Bearer {self._acp_token}"}, + ) + from code4me2_agent.config import ServerAgentConfig + + try: + self._server_agent_config = ServerAgentConfig.from_managed_payload(payload) + except ValueError as exc: + # The ACP client only sees "authorization rejected"; keep the reason + # (for example a server older than this runtime) in the agent log. + logger.warning("Refused the server's managed-agent policy: %s", exc) + raise AcpAuthorizationFailure( + "The Code4Me server returned an invalid managed-agent policy." + ) from exc diff --git a/src/code4me2_agent/session_state.py b/src/code4me2_agent/session_state.py new file mode 100644 index 0000000..2663156 --- /dev/null +++ b/src/code4me2_agent/session_state.py @@ -0,0 +1,318 @@ +"""Per-session tool state that must outlive ToolRegistry rebuilds. + +Managed prompts re-apply the run policy before every prompt, which rebuilds the +file tools and the registry; anything the agent should remember for the whole +chat (which files the model has looked at, what each turn changed) therefore +lives here, owned by ``EchoAgentCore``. Nothing is written into the project: +checkpoints are kept in memory only. +""" + +from __future__ import annotations + +import os +from contextlib import contextmanager +from dataclasses import dataclass, field +from difflib import unified_diff +from pathlib import Path +from threading import RLock +from typing import Any, Iterator + +# Checkpoint limits: files above this are recorded as not restorable, and the +# oldest turns are dropped once the total or the turn count is exceeded. +MAX_CHECKPOINT_FILE_CHARS = 2 * 1024 * 1024 +MAX_CHECKPOINT_TOTAL_CHARS = 32 * 1024 * 1024 +MAX_CHECKPOINT_TURNS = 20 + + +@dataclass(frozen=True) +class FileChange: + """One successful workspace mutation reported by the file tools. + + ``before``/``after`` are the text before and after the change; ``None`` + means the path did not exist (before) or no longer exists (after). + ``before_known`` is false when the previous content could not be captured + (binary, oversized, a directory), which makes the change not restorable. + """ + + path: str + before: str | None + after: str | None + before_known: bool = True + + +@dataclass +class _PathRecord: + before: str | None + after: str | None + before_known: bool + order: int + + +@dataclass +class TurnCheckpoint: + turn_id: str + prompt_preview: str + files: dict[str, _PathRecord] = field(default_factory=dict) + + def size(self) -> int: + total = 0 + for record in self.files.values(): + total += len(record.before or "") + len(record.after or "") + return total + + +@dataclass(frozen=True) +class UndoOutcome: + restored: tuple[str, ...] = () + deleted: tuple[str, ...] = () + skipped: tuple[tuple[str, str], ...] = () + turn_id: str | None = None + + @property + def nothing_to_undo(self) -> bool: + return self.turn_id is None + + +def normalize_workspace_path(workspace_root: Path, path: object) -> str | None: + """Workspace-relative posix path for ``path``, or None when it escapes.""" + if not isinstance(path, str) or not path.strip(): + return None + root = workspace_root.resolve() + candidate = Path(path.strip()).expanduser() + if not candidate.is_absolute(): + candidate = root / candidate + try: + resolved = Path(os.path.normpath(str(candidate))) + if resolved.exists() or resolved.is_symlink(): + resolved = resolved.resolve() + relative = resolved.relative_to(root) + except (OSError, ValueError): + return None + text = relative.as_posix() + return text if text != "." else None + + +class SessionToolState: + def __init__(self) -> None: + self._lock = RLock() + self._seen: set[str] = set() + self._turns: list[TurnCheckpoint] = [] + self._current: TurnCheckpoint | None = None + self._order = 0 + self._recording_suspended = 0 + + # ------------------------------------------------------------ seen files + + def mark_seen(self, path: str | None) -> None: + if path: + with self._lock: + self._seen.add(path) + + def has_seen(self, path: str | None) -> bool: + if not path: + return False + with self._lock: + return path in self._seen + + def forget_seen(self) -> None: + with self._lock: + self._seen.clear() + + # ------------------------------------------------------------ checkpoints + + def begin_turn(self, turn_id: str, prompt: str) -> None: + with self._lock: + self._current = TurnCheckpoint( + turn_id=turn_id, prompt_preview=" ".join(prompt.split())[:120] + ) + + def end_turn(self) -> None: + with self._lock: + current = self._current + self._current = None + if current is None or not current.files: + return + self._turns.append(current) + while len(self._turns) > MAX_CHECKPOINT_TURNS or ( + len(self._turns) > 1 + and sum(turn.size() for turn in self._turns) > MAX_CHECKPOINT_TOTAL_CHARS + ): + self._turns.pop(0) + + @contextmanager + def recording_suspended(self) -> Iterator[None]: + with self._lock: + self._recording_suspended += 1 + try: + yield + finally: + with self._lock: + self._recording_suspended -= 1 + + def record_change(self, change: FileChange) -> None: + """File-tool observer: remember the first before-state per path per turn.""" + with self._lock: + if self._recording_suspended or self._current is None: + return + before = change.before + before_known = change.before_known + if before is not None and len(before) > MAX_CHECKPOINT_FILE_CHARS: + before, before_known = None, False + after = change.after + if after is not None and len(after) > MAX_CHECKPOINT_FILE_CHARS: + after, before_known = None, False + record = self._current.files.get(change.path) + if record is None: + self._order += 1 + self._current.files[change.path] = _PathRecord( + before=before, after=after, before_known=before_known, order=self._order + ) + else: + record.after = after + if not before_known: + record.before_known = False + + def changed_paths(self) -> list[str]: + """Paths changed so far in the current turn, in first-change order.""" + with self._lock: + if self._current is None: + return [] + return [ + path + for path, _record in sorted( + self._current.files.items(), key=lambda item: item[1].order + ) + ] + + def turn_diff(self, *, max_chars: int = 40_000) -> str: + """Unified diff of the current turn's changes (for self-review).""" + with self._lock: + records = ( + sorted(self._current.files.items(), key=lambda item: item[1].order) + if self._current is not None + else [] + ) + return _render_diff(records, max_chars=max_chars) + + def session_diff(self, *, max_chars: int = 40_000) -> str: + """Unified diff from each path's earliest recorded state to its latest.""" + with self._lock: + merged: dict[str, _PathRecord] = {} + turns = list(self._turns) + ([self._current] if self._current is not None else []) + for turn in turns: + for path, record in sorted(turn.files.items(), key=lambda item: item[1].order): + existing = merged.get(path) + if existing is None: + merged[path] = _PathRecord( + record.before, record.after, record.before_known, record.order + ) + else: + existing.after = record.after + existing.before_known = existing.before_known and record.before_known + return _render_diff(sorted(merged.items(), key=lambda item: item[1].order), max_chars=max_chars) + + def checkpoint_count(self) -> int: + with self._lock: + return len(self._turns) + + def last_turn(self) -> TurnCheckpoint | None: + with self._lock: + return self._turns[-1] if self._turns else None + + def undo_last_turn(self, file_tools: Any) -> UndoOutcome: + """Restore every file the last changing turn touched. + + A file is restored only while its current content is still exactly what + the agent left behind; anything the user edited since is skipped and + reported, never overwritten. + """ + with self._lock: + if not self._turns: + return UndoOutcome() + turn = self._turns.pop() + restored: list[str] = [] + deleted: list[str] = [] + skipped: list[tuple[str, str]] = [] + with self.recording_suspended(): + for path, record in sorted( + turn.files.items(), key=lambda item: item[1].order, reverse=True + ): + if not record.before_known: + skipped.append((path, "its previous content was not captured")) + continue + try: + current = _current_text(file_tools, path) + except Exception as exc: # noqa: BLE001 - unreadable now: leave it alone + skipped.append((path, f"it could not be read ({exc})")) + continue + if current != record.after: + skipped.append((path, "it changed after the agent's edit")) + continue + try: + if record.before is None: + if current is not None: + _delete(file_tools, path) + deleted.append(path) + else: + _write(file_tools, path, record.before) + restored.append(path) + except Exception as exc: # noqa: BLE001 - report, keep going + skipped.append((path, f"restoring failed: {exc}")) + return UndoOutcome( + restored=tuple(restored), + deleted=tuple(deleted), + skipped=tuple(skipped), + turn_id=turn.turn_id, + ) + + +def _render_diff(records: list[tuple[str, _PathRecord]], *, max_chars: int) -> str: + parts: list[str] = [] + for path, record in records: + if not record.before_known: + parts.append(f"--- {path}\n+++ {path}\n(binary or oversized file changed; diff not available)\n") + continue + before = record.before or "" + after = record.after or "" + if before == after and record.before is not None and record.after is not None: + continue + diff = "".join( + unified_diff( + before.splitlines(keepends=True), + after.splitlines(keepends=True), + fromfile=f"a/{path}" if record.before is not None else "/dev/null", + tofile=f"b/{path}" if record.after is not None else "/dev/null", + ) + ) + if diff and not diff.endswith("\n"): + diff += "\n" + parts.append(diff) + text = "".join(parts) + if len(text) > max_chars: + text = text[:max_chars] + f"\n[diff truncated: {len(text) - max_chars} more characters]\n" + return text + + +def _current_text(file_tools: Any, path: str) -> str | None: + read_text = getattr(file_tools, "read_text", None) + if not callable(read_text): + return None + try: + text, _replaced = read_text(path, strict=True, tool_name="undo") + except FileNotFoundError: + return None + except Exception as exc: # noqa: BLE001 + if getattr(exc, "code", None) == "file_not_found": + return None + raise + return text + + +def _write(file_tools: Any, path: str, content: str) -> None: + write = getattr(file_tools, "restore_text", None) or getattr(file_tools, "write_file") + write(path, content) + + +def _delete(file_tools: Any, path: str) -> None: + discard = getattr(file_tools, "discard_file", None) or getattr(file_tools, "delete_file") + discard(path) diff --git a/src/code4me2_agent/slash_commands.py b/src/code4me2_agent/slash_commands.py new file mode 100644 index 0000000..aae6e7c --- /dev/null +++ b/src/code4me2_agent/slash_commands.py @@ -0,0 +1,63 @@ +"""Slash commands the runtime answers itself (ACP ``available_commands_update``). + +Clients that render ACP commands send them as ordinary prompt text starting +with ``/``; typing the same text works in any client, so the commands are +parsed from the prompt rather than from a dedicated request. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SlashCommand: + name: str + description: str + hint: str | None = None + + +COMMANDS: tuple[SlashCommand, ...] = ( + SlashCommand("status", "Show this session's status"), + SlashCommand( + "undo", + "Revert the file changes of the most recent turn that changed files", + ), + SlashCommand( + "compact", + "Summarise the earlier conversation now to free context space", + ), + SlashCommand( + "review", + "Review every change made in this session for bugs and missing verification", + hint="optional focus, e.g. error handling", + ), +) +_BY_NAME = {command.name: command for command in COMMANDS} +# Study (managed) sessions offer only commands that neither reveal nor change +# the assigned arm; there "/review" or "/compact" is an ordinary request. +MANAGED_COMMAND_NAMES = frozenset({"status", "undo"}) + + +def available(*, managed: bool) -> tuple[SlashCommand, ...]: + if not managed: + return COMMANDS + return tuple(command for command in COMMANDS if command.name in MANAGED_COMMAND_NAMES) + + +@dataclass(frozen=True) +class ParsedCommand: + name: str + argument: str + + +def parse(prompt: str, *, managed: bool = False) -> ParsedCommand | None: + """The command in ``prompt``, or None when it is an ordinary message.""" + text = (prompt or "").strip() + if not text.startswith("/") or "\n" in text: + return None + name, _space, argument = text[1:].partition(" ") + name = name.strip().lower() + if name not in {command.name for command in available(managed=managed)}: + return None + return ParsedCommand(name=name, argument=argument.strip()) diff --git a/src/code4me2_agent/syntax_check.py b/src/code4me2_agent/syntax_check.py new file mode 100644 index 0000000..1a8a5c8 --- /dev/null +++ b/src/code4me2_agent/syntax_check.py @@ -0,0 +1,99 @@ +"""Cheap syntax checks after an edit (Python, JSON, TOML). + +The edit is always applied; a syntax error the edit introduced is attached to +the tool result so the model fixes it on its next step instead of discovering +it much later. Files that were already broken, or that use syntax this parser +does not know (a newer Python, JSON with comments), are left alone: only a +transition from parsing to not parsing is reported. +""" + +from __future__ import annotations + +import ast +import json +import sys +import warnings +from dataclasses import dataclass + +try: # Python 3.11+ + import tomllib +except ImportError: # pragma: no cover - the bundle ships 3.13 + tomllib = None # type: ignore[assignment] + +_MAX_CHECK_CHARS = 2 * 1024 * 1024 + + +@dataclass(frozen=True) +class SyntaxProblem: + language: str + line: int | None + message: str + + def as_result(self) -> dict[str, object]: + return {"language": self.language, "line": self.line, "message": self.message} + + +def _language(path: str) -> str | None: + lowered = path.lower() + if lowered.endswith((".py", ".pyi")): + return "python" + if lowered.endswith(".json"): + return "json" + if lowered.endswith(".toml"): + return "toml" + return None + + +def _problem(language: str, text: str) -> SyntaxProblem | None: + if language == "python": + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + ast.parse(text) + except SyntaxError as exc: + version = f"{sys.version_info.major}.{sys.version_info.minor}" + return SyntaxProblem( + "python", + exc.lineno, + f"{exc.msg} (Python {version} parser)", + ) + except (ValueError, RecursionError, MemoryError): + return None + return None + if language == "json": + if not text.strip(): + return None + try: + json.loads(text) + except json.JSONDecodeError as exc: + return SyntaxProblem("json", exc.lineno, exc.msg) + except RecursionError: + return None + return None + if language == "toml": + if tomllib is None: + return None + try: + tomllib.loads(text) + except tomllib.TOMLDecodeError as exc: + line = getattr(exc, "lineno", None) + message = getattr(exc, "msg", None) or str(exc) + return SyntaxProblem("toml", line, message) + return None + return None + + +def introduced_syntax_problem(path: str, before: str | None, after: str) -> SyntaxProblem | None: + """The syntax problem ``after`` has that ``before`` did not, if any. + + ``before=None`` means a new file: any problem is new. + """ + language = _language(path) + if language is None or len(after) > _MAX_CHECK_CHARS: + return None + problem = _problem(language, after) + if problem is None: + return None + if before is not None and len(before) <= _MAX_CHECK_CHARS and _problem(language, before) is not None: + return None + return problem diff --git a/src/code4me2_agent/telemetry.py b/src/code4me2_agent/telemetry.py new file mode 100644 index 0000000..ad3f419 --- /dev/null +++ b/src/code4me2_agent/telemetry.py @@ -0,0 +1,319 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from threading import RLock +from typing import TYPE_CHECKING, Any, Callable, Protocol +from urllib import request +from uuid import uuid4 + +from code4me2_agent.tls import USER_AGENT, https_context + +if TYPE_CHECKING: + from pathlib import Path + + from code4me2_agent.config import AgentConfig + + +SCHEMA_VERSION = "code4me.agent.event.v1" + +_CONTENT_PAYLOAD_KEYS = frozenset( + { + "arguments", + "argv", + "content", + "edits", + "entries", + "error_message", + "glob", + "messages", + "new_text", + "old_text", + "pattern", + "query", + "raw_arguments", + "request", + "response", + "result", + "stderr", + "stdout", + "text", + } +) + + +def _payload_without_content(payload: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in payload.items() + if key not in _CONTENT_PAYLOAD_KEYS + } + +# TODO this file should be working with the database. and may be moved to another folder. +class TelemetrySink(Protocol): + def append(self, event: dict[str, Any]) -> None: + ... + + +class UploadEventsCallable(Protocol): + def __call__( + self, + run: dict[str, Any], + events: list[dict[str, Any]], + auth_headers: dict[str, str], + ) -> None: + ... + + +class JsonlTelemetrySink: + def __init__(self, trace_path: Path) -> None: + self._trace_path = trace_path + + def append(self, event: dict[str, Any]) -> None: + self._trace_path.parent.mkdir(parents=True, exist_ok=True) + with self._trace_path.open("a", encoding="utf-8") as trace_file: + trace_file.write(json.dumps(event, sort_keys=True)) + trace_file.write("\n") + + +JsonlTraceRecorder = JsonlTelemetrySink + + +class ServerUploadTelemetrySink: + def __init__( + self, + upload_events: UploadEventsCallable, + batch_size: int = 50, + auth_headers: dict[str, str] | None = None, + auth_headers_provider: Callable[[], dict[str, str]] | None = None, + ) -> None: + self._upload_events = upload_events + self._batch_size = max(1, int(batch_size)) + self._auth_headers = dict(auth_headers or {}) + self._auth_headers_provider = auth_headers_provider + self._pending_run_events: dict[str, list[dict[str, Any]]] = {} + + def set_auth_headers(self, auth_headers: dict[str, str]) -> None: + self._auth_headers = dict(auth_headers) + + def append(self, event: dict[str, Any]) -> None: + run_id = event["run_id"] + pending_events = self._pending_run_events.setdefault(run_id, []) + pending_events.append(event) + self._flush_ready_runs() + + def _flush_ready_runs(self) -> None: + for run_id in list(self._pending_run_events.keys()): + pending_events = self._pending_run_events[run_id] + if not self._should_flush(pending_events): + continue + + run_payload = _build_run_payload(pending_events) + try: + headers = ( + self._auth_headers_provider() + if callable(self._auth_headers_provider) + else dict(self._auth_headers) + ) + self._upload_events(run_payload, pending_events, dict(headers)) + except Exception: + continue + del self._pending_run_events[run_id] + + def _should_flush(self, pending_events: list[dict[str, Any]]) -> bool: + if len(pending_events) >= self._batch_size: + return True + return any(event["event_type"] == "agent.run.completed" for event in pending_events) + + +def _build_run_payload(events: list[dict[str, Any]]) -> dict[str, Any]: + first_event = events[0] + status = "in_progress" + started_at = None + completed_at = None + + for event in events: + if event["event_type"] == "agent.run.started": + status = str(event.get("payload", {}).get("status", "started")) + started_at = event["timestamp"] + elif event["event_type"] == "agent.run.completed": + status = str(event.get("payload", {}).get("status", "completed")) + completed_at = event["timestamp"] + + return { + "run_id": first_event["run_id"], + "session_id": first_event["session_id"], + "status": status, + "source": first_event.get("source", "code4me2_agent"), + "started_at": started_at, + "completed_at": completed_at, + } + + +def upload_event_batch_http( + run: dict[str, Any], + events: list[dict[str, Any]], + auth_headers: dict[str, str], + *, + ingest_url: str, + timeout_seconds: float, +) -> None: + payload = {"run": run, "events": events} + payload_bytes = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json", "User-Agent": USER_AGENT} + headers.update(auth_headers) + http_request = request.Request( + ingest_url, + data=payload_bytes, + headers=headers, + method="POST", + ) + with request.urlopen(http_request, timeout=timeout_seconds, context=https_context()) as response: + if response.status >= 400: + raise RuntimeError(f"Telemetry upload failed with HTTP {response.status}.") + + +def build_upload_sink(config: AgentConfig) -> ServerUploadTelemetrySink | None: + if not config.upload.enabled or config.upload.ingest_url is None: + return None + + def upload_events( + run: dict[str, Any], + events: list[dict[str, Any]], + auth_headers: dict[str, str], + ) -> None: + upload_event_batch_http( + run=run, + events=events, + auth_headers=auth_headers, + ingest_url=config.upload.ingest_url, + timeout_seconds=config.upload.timeout_seconds, + ) + + return ServerUploadTelemetrySink( + upload_events=upload_events, + batch_size=config.upload.batch_size, + auth_headers=config.upload.auth_headers, + auth_headers_provider=config.upload.auth_headers_provider, + ) + + +class AgentTelemetryRecorder: + def __init__(self, config: AgentConfig, sinks: list[TelemetrySink] | None = None) -> None: + self._config = config + # Participant mode must not leave study prompts, tool arguments, or a + # surprise .code4me directory in the participant's project. The + # authenticated server sink remains the telemetry system of record; + # local JSONL is retained only for explicit developer configurations. + self._jsonl = None if config.managed_mode else JsonlTelemetrySink(config.trace_path) + if sinks is None: + upload_sink = build_upload_sink(config) + self._sinks = [upload_sink] if upload_sink is not None else [] + else: + self._sinks = sinks + self._run_sequences: dict[str, int] = {} + # Read-only tool calls run in parallel: sequence numbers and sink + # writes must stay ordered and whole. + self._lock = RLock() + + def apply_config(self, config: AgentConfig) -> None: + """Refresh policy metadata and rotating bearer headers without dropping events.""" + self._config = config + for sink in self._sinks: + set_auth_headers = getattr(sink, "set_auth_headers", None) + if callable(set_auth_headers): + set_auth_headers(config.upload.auth_headers) + + def record( + self, + *, + event_type: str, + run_id: str, + request_id: str, + parent_event_id: str | None, + payload: dict[str, Any], + message_id: str | None = None, + metrics: dict[str, Any] | None = None, + contains_user_prompt: bool = False, + contains_agent_response: bool = False, + raw_payload: dict[str, Any] | None = None, + ) -> dict[str, Any]: + with self._lock: + return self._record_locked( + event_type=event_type, + run_id=run_id, + request_id=request_id, + parent_event_id=parent_event_id, + payload=payload, + message_id=message_id, + metrics=metrics, + contains_user_prompt=contains_user_prompt, + contains_agent_response=contains_agent_response, + raw_payload=raw_payload, + ) + + def _record_locked( + self, + *, + event_type: str, + run_id: str, + request_id: str, + parent_event_id: str | None, + payload: dict[str, Any], + message_id: str | None, + metrics: dict[str, Any] | None, + contains_user_prompt: bool, + contains_agent_response: bool, + raw_payload: dict[str, Any] | None, + ) -> dict[str, Any]: + sequence = self._run_sequences.get(run_id, 0) + 1 + self._run_sequences[run_id] = sequence + + observed_tool = str(payload.get("tool_name", "none")) + stored_payload = ( + payload + if self._config.store_agent_content + else _payload_without_content(payload) + ) + event = { + "schema_version": SCHEMA_VERSION, + "event_id": uuid4().hex, + "event_type": event_type, + "timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "sequence": sequence, + "source": "code4me2_agent", + "session_id": self._config.session_id, + "run_id": run_id, + "request_id": request_id, + "parent_event_id": parent_event_id, + "payload": stored_payload, + "raw_payload": ( + raw_payload + if self._config.raw_capture_enabled + and self._config.store_agent_content + else None + ), + "metrics": metrics or {}, + "privacy": { + "capture_mode": "raw" if self._config.raw_capture_enabled else "redacted", + "raw_capture_enabled": self._config.raw_capture_enabled, + "contains_user_prompt": contains_user_prompt, + "contains_agent_response": contains_agent_response, + "redaction_applied": not self._config.raw_capture_enabled, + }, + "observability": { + "level": "full", + "adapter": self._config.adapter.name, + "llm": self._config.adapter.name, + "tools": observed_tool, + "missing_internals": [], + }, + } + if message_id is not None: + event["message_id"] = message_id + + if self._jsonl is not None: + self._jsonl.append(event) + for sink in self._sinks: + sink.append(event) + return event diff --git a/src/code4me2_agent/tls.py b/src/code4me2_agent/tls.py new file mode 100644 index 0000000..23ef505 --- /dev/null +++ b/src/code4me2_agent/tls.py @@ -0,0 +1,37 @@ +"""TLS trust for the agent's own HTTPS calls to the backend. + +Inside the packaged (PyInstaller) runtime, Python's compiled-in OpenSSL CA +paths do not exist on the participant's machine, so on macOS a plain +``urlopen`` has an empty trust store and fails every HTTPS backend with +``CERTIFICATE_VERIFY_FAILED``. The bundled certifi roots are added on top of +the platform defaults, which still honour ``SSL_CERT_FILE`` and the system +store wherever Python can read it. + +Every backend call also names the agent in ``User-Agent``: the Cloudflare edge +in front of the backend rejects urllib's default ``Python-urllib/3.x`` with +403 (error 1010) before the request reaches the server. +""" + +from __future__ import annotations + +import ssl +from functools import lru_cache + +from code4me2_agent._build_version import RUNTIME_VERSION + +USER_AGENT = f"code4me2-agent/{RUNTIME_VERSION}" + + +@lru_cache(maxsize=1) +def https_context() -> ssl.SSLContext: + """A verifying client context: platform trust plus the bundled certifi roots.""" + context = ssl.create_default_context() + try: + import certifi + + context.load_verify_locations(cafile=certifi.where()) + except (ImportError, OSError): + # Without certifi (or its bundle file) the platform defaults are all + # there is; verification stays on either way. + pass + return context diff --git a/src/code4me2_agent/tool_catalog.py b/src/code4me2_agent/tool_catalog.py new file mode 100644 index 0000000..2452490 --- /dev/null +++ b/src/code4me2_agent/tool_catalog.py @@ -0,0 +1,512 @@ +"""Model-facing tool catalogue: schemas, ACP kinds and approval classes. + +This is the runtime half of the contract whose server half is +``src/agents/tools.py`` (``CODE4ME2_AGENT_TOOLS``). A profile freezes tool +*names*; the runtime advertises the schema for each allowed name on every turn. +Keep names stable: existing study profiles pin them. +""" + +from __future__ import annotations + +from typing import Any + +# ACP ToolKind per tool (acp.schema.ToolKind). ``update_plan`` has no card at +# all: the ACP ``plan`` update renders natively and a card would duplicate it. +TOOL_KINDS: dict[str, str] = { + "read_file": "read", + "create_file": "edit", + "write_file": "edit", + "replace_text": "edit", + "edit_file": "edit", + "apply_patch": "edit", + "delete_file": "delete", + "move_file": "move", + "list_files": "search", + "glob_files": "search", + "grep_files": "search", + "search_files": "search", + "run_command": "execute", +} + +# Tools that change the workspace or run code; these need approval under +# ``per_step`` and are hard-denied under ``suggestion_only``. +MUTATING_TOOLS: frozenset[str] = frozenset( + { + "create_file", + "write_file", + "replace_text", + "edit_file", + "apply_patch", + "delete_file", + "move_file", + "run_command", + } +) + +# Approval scope class used for "allow for session" decisions. +APPROVAL_KINDS: dict[str, str] = { + "run_command": "execute", + "create_file": "edit", + "write_file": "edit", + "replace_text": "edit", + "edit_file": "edit", + "apply_patch": "edit", + "delete_file": "edit", + "move_file": "edit", +} + +# ``update_plan`` renders as the ACP plan and ``ask_user`` as the agent's final +# message, so neither gets a tool card. +NO_CARD_TOOLS: frozenset[str] = frozenset({"update_plan", "ask_user"}) + +DEFAULT_IGNORED_DIRS_TEXT = ( + ".git, node_modules, build, dist, target, out, .idea, .gradle, .venv, __pycache__" +) + + +def tool_kind(name: str) -> str: + if name.startswith("mcp__"): + return "other" + return TOOL_KINDS.get(name, "other") + + +def approval_kind(name: str) -> str: + if name.startswith("mcp__"): + return "other" + return APPROVAL_KINDS.get(name, "other") + + +def requires_manual_approval(name: str) -> bool: + return name in MUTATING_TOOLS or name.startswith("mcp__") + + +def is_read_only(name: str) -> bool: + return not requires_manual_approval(name) + + +_PATH_DESCRIPTION = ( + "Workspace-relative path (preferred, e.g. src/app.py) or an absolute path inside the workspace." +) + + +def _function_tool( + name: str, + description: str, + properties: dict[str, Any], + required: list[str], +) -> dict[str, Any]: + return { + "type": "function", + "function": { + "name": name, + "description": description, + "parameters": { + "type": "object", + "properties": properties, + "required": required, + "additionalProperties": False, + }, + }, + } + + +def _path_property(description: str = _PATH_DESCRIPTION) -> dict[str, Any]: + return {"type": "string", "description": description} + + +def _force_property() -> dict[str, Any]: + return { + "type": "boolean", + "description": ( + "Skip the read-before-edit check for a file you have not read in this session. " + "Only when you are certain of its current content. Default false." + ), + } + + +APPLY_PATCH_DESCRIPTION = ( + "Edit, create, delete or rename files with one patch; all changes apply together or none do. " + "Format:\n" + "*** Begin Patch\n" + "*** Update File: src/app.py\n" + "@@ def handler(event):\n" + "- return None\n" + "+ return event\n" + "*** Add File: src/new.py\n" + '+print("hello")\n' + "*** Delete File: src/old.py\n" + "*** End Patch\n" + "Rules: paths are workspace-relative. In an update, '@@' starts a chunk and may name a line " + "to find first (a def/class/function signature) so the chunk is placed after it; then lines " + "starting with ' ' are unchanged context (include about 3 lines before and after each " + "change), '-' lines are removed and '+' lines are added. Context must match the current file " + "(read it first). '*** Move to: new/path' right after '*** Update File:' renames the file; " + "'*** End of File' after a chunk pins it to the end of the file. Every line of an added file " + "starts with '+'." +) + + +def tool_definitions() -> list[dict[str, Any]]: + return [ + _function_tool( + "read_file", + ( + "Read a UTF-8 text file in the workspace. Each output line is prefixed with its " + "1-based line number and a pipe, e.g. `12|code`; the prefix is display-only and " + "must never be copied into old_text for replace_text or edit_file. Long files are " + "paged: the result reports total_lines, truncated and next_offset; call again with " + "offset=next_offset to continue. Lines longer than 2000 characters are cut with a " + "marker. Binary files are rejected. Use glob_files or grep_files to locate files first." + ), + { + "path": _path_property(), + "offset": { + "type": "integer", + "minimum": 1, + "description": "1-based line number to start from. Default 1.", + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 5000, + "description": "Maximum number of lines to return. Default 1000.", + }, + }, + ["path"], + ), + _function_tool( + "create_file", + ( + "Create a new UTF-8 text file with the given content. Fails if the file already " + "exists: use write_file to overwrite it, or replace_text/edit_file to change part " + "of it. Parent directories are created." + ), + { + "path": _path_property(), + "content": {"type": "string", "description": "Complete file content."}, + }, + ["path", "content"], + ), + _function_tool( + "write_file", + ( + "Overwrite a UTF-8 text file with the complete new content, creating it and its " + "parent directories if needed. Prefer replace_text or edit_file for partial " + "changes to an existing file so unrelated lines are never lost." + ), + { + "path": _path_property(), + "content": {"type": "string", "description": "Complete new file content."}, + "force": _force_property(), + }, + ["path", "content"], + ), + _function_tool( + "replace_text", + ( + "Replace one occurrence of old_text with new_text in a file you have read. Copy " + "old_text exactly from read_file, without the line-number prefix. It must match " + "exactly one place: if it matches several the call fails and reports their line " + "numbers (add surrounding lines, or set replace_all=true to change every exact " + "occurrence). Differences only in trailing whitespace, indentation or line endings " + "are tolerated and reported; new_text is re-indented to fit." + ), + { + "path": _path_property(), + "old_text": { + "type": "string", + "description": "Exact text to find (copied from read_file, without the N| prefix).", + }, + "new_text": {"type": "string", "description": "Replacement text."}, + "replace_all": { + "type": "boolean", + "description": "Replace every occurrence instead of requiring a unique match. Default false.", + }, + "force": _force_property(), + }, + ["path", "old_text", "new_text"], + ), + _function_tool( + "edit_file", + ( + "Apply several exact text replacements to one file as a single atomic operation. " + "Edits are applied in order, so later edits see the result of earlier ones; each " + "old_text must match exactly once unless its replace_all is true. If any edit " + "fails, nothing is written and the error names the failing edit. Prefer this over " + "several replace_text calls on the same file." + ), + { + "path": _path_property(), + "edits": { + "type": "array", + "minItems": 1, + "maxItems": 50, + "description": "Replacements to apply in order.", + "items": { + "type": "object", + "properties": { + "old_text": {"type": "string", "description": "Exact text to find."}, + "new_text": {"type": "string", "description": "Replacement text."}, + "replace_all": { + "type": "boolean", + "description": "Replace every occurrence of this old_text. Default false.", + }, + }, + "required": ["old_text", "new_text"], + }, + }, + "force": _force_property(), + }, + ["path", "edits"], + ), + _function_tool( + "apply_patch", + APPLY_PATCH_DESCRIPTION, + { + "patch": { + "type": "string", + "description": "The complete patch, from '*** Begin Patch' to '*** End Patch'.", + }, + "force": _force_property(), + }, + ["patch"], + ), + _function_tool( + "delete_file", + ( + "Delete a file or an empty directory inside the workspace. Non-empty directories " + "are refused. This cannot be undone." + ), + {"path": _path_property()}, + ["path"], + ), + _function_tool( + "move_file", + ( + "Move or rename a file or directory inside the workspace. Destination parent " + "directories are created. Fails if the destination already exists unless " + "overwrite=true (files only). Use this instead of read + create + delete." + ), + { + "source_path": _path_property("Current path of the file or directory."), + "destination_path": _path_property("New path."), + "overwrite": { + "type": "boolean", + "description": "Replace an existing destination file. Default false.", + }, + }, + ["source_path", "destination_path"], + ), + _function_tool( + "list_files", + ( + "List the entries of a directory. Non-recursive by default; directory names end " + "with '/'. Set recursive=true (optionally with max_depth) to walk the tree. Common " + f"build, VCS and dependency directories ({DEFAULT_IGNORED_DIRS_TEXT}, ...) are " + "skipped unless include_ignored=true. Results are sorted and capped. Use " + "glob_files to find files by name pattern." + ), + { + "path": _path_property("Directory to list. Default '.' (the workspace root)."), + "recursive": { + "type": "boolean", + "description": "Walk subdirectories. Default false.", + }, + "max_depth": { + "type": "integer", + "minimum": 1, + "maximum": 20, + "description": "With recursive=true, the maximum directory depth. Default unlimited.", + }, + "max_results": { + "type": "integer", + "minimum": 1, + "maximum": 2000, + "description": "Maximum entries to return. Default 500.", + }, + "include_ignored": { + "type": "boolean", + "description": "Also list build/VCS/dependency directories. Default false.", + }, + }, + [], + ), + _function_tool( + "glob_files", + ( + "Find files whose path relative to `path` matches a glob pattern, e.g. '**/*.kt', " + "'src/main/**/Service*.java' or '*.md' (which matches only directly under path). " + "'**' spans directories. Results are workspace-relative, sorted by path and capped; " + f"common build, VCS and dependency directories ({DEFAULT_IGNORED_DIRS_TEXT}, ...) " + "are skipped unless include_ignored=true." + ), + { + "pattern": { + "type": "string", + "description": "Glob pattern relative to path; '**' matches across directories.", + }, + "path": _path_property("Directory to search in. Default '.' (the workspace root)."), + "max_results": { + "type": "integer", + "minimum": 1, + "maximum": 2000, + "description": "Maximum files to return. Default 500.", + }, + "include_ignored": { + "type": "boolean", + "description": "Also search build/VCS/dependency directories. Default false.", + }, + }, + ["pattern"], + ), + _function_tool( + "grep_files", + ( + "Search file contents with a Python regular expression (escape ( ) . [ ] + ? * for " + "literal text). output_mode 'content' (default) returns matching lines with path " + "and line number plus optional context lines; 'files_with_matches' returns only " + "the paths; 'count' returns per-file match counts. Binary files, files over 2 MiB " + f"and common build, VCS and dependency directories ({DEFAULT_IGNORED_DIRS_TEXT}, " + "...) are skipped. Results are capped (truncated=true when the cap is hit): narrow " + "with path or glob, or use count mode." + ), + { + "pattern": { + "type": "string", + "description": "Python regular expression to search for.", + }, + "path": _path_property("File or directory to search. Default '.' (the workspace root)."), + "glob": { + "type": "string", + "description": "Only search files whose path relative to `path` matches this glob, e.g. '**/*.java'.", + }, + "case_insensitive": { + "type": "boolean", + "description": "Ignore case. Default false.", + }, + "context_lines": { + "type": "integer", + "minimum": 0, + "maximum": 10, + "description": "Lines of context before and after each match (content mode). Default 0.", + }, + "max_results": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "Maximum matches (content mode) or files to return. Default 200.", + }, + "output_mode": { + "type": "string", + "enum": ["content", "files_with_matches", "count"], + "description": "What to return. Default 'content'.", + }, + "include_ignored": { + "type": "boolean", + "description": "Also search build/VCS/dependency directories. Default false.", + }, + }, + ["pattern"], + ), + _function_tool( + "search_files", + ( + "Legacy literal, case-sensitive text search; prefer grep_files. Equivalent to " + "grep_files with the query escaped as a literal pattern." + ), + { + "query": {"type": "string", "description": "Literal text to find."}, + "path": _path_property("File or directory to search. Default '.' (the workspace root)."), + }, + ["query"], + ), + _function_tool( + "run_command", + ( + "Run one program inside the workspace without a shell and return exit_code, " + "stdout, stderr, duration_ms and timed_out. argv[0] is a program installed on this " + "machine or the path of a script in the project (resolved against cwd); programs " + "the study blocks are refused. There is no shell: pipes, globs, " + "'&&', 'cd' and redirection are not available; pass arguments as separate argv " + "items. stdin is closed. Output is captured with head and tail truncation " + "(output_truncated=true). Set timeout_seconds for long builds or test runs (max 600)." + ), + { + "argv": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + "description": "Program name followed by its arguments, e.g. [\"pytest\", \"-q\", \"tests\"].", + }, + "cwd": _path_property("Working directory. Default '.' (the workspace root)."), + "timeout_seconds": { + "type": "integer", + "minimum": 1, + "maximum": 600, + "description": "Kill the command after this many seconds. Default 120.", + }, + }, + ["argv"], + ), + _function_tool( + "ask_user", + ( + "Ask the user a question and end your turn; their answer arrives as their next " + "message. Use it only when a decision is genuinely ambiguous and would change the " + "outcome, or you need information only the user has. Never use it for routine " + "confirmation. Offer short answer options when the choice is between known " + "alternatives." + ), + { + "question": {"type": "string", "description": "One clear, specific question."}, + "options": { + "type": "array", + "items": {"type": "string"}, + "maxItems": 6, + "description": "Optional short answer choices, most likely first.", + }, + }, + ["question"], + ), + _function_tool( + "update_plan", + ( + "Publish or update your task plan so the user can follow progress. Send the " + "complete list every time (it replaces the previous plan); keep at most one entry " + "in_progress and mark finished steps completed. Use it for multi-step tasks before " + "starting and whenever a step changes state. It has no effect on files." + ), + { + "entries": { + "type": "array", + "minItems": 1, + "maxItems": 30, + "description": "The full plan, in order.", + "items": { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "Short description of the step.", + }, + "status": { + "type": "string", + "enum": ["pending", "in_progress", "completed"], + }, + "priority": { + "type": "string", + "enum": ["high", "medium", "low"], + "description": "Default medium.", + }, + }, + "required": ["content", "status"], + }, + }, + }, + ["entries"], + ), + ] + + +def tool_names() -> list[str]: + return [definition["function"]["name"] for definition in tool_definitions()] diff --git a/src/code4me2_agent/tool_errors.py b/src/code4me2_agent/tool_errors.py new file mode 100644 index 0000000..c01244a --- /dev/null +++ b/src/code4me2_agent/tool_errors.py @@ -0,0 +1,86 @@ +"""Typed, model-facing tool failures. + +Every ``ToolError`` carries a stable ``code`` that the agent loop copies into the +tool result as ``error_code`` so the model can react to the failure class, while +``str(exc)`` is the human/model readable message. Errors that correspond to a +builtin also inherit from it so existing ``except PermissionError`` / +``except FileNotFoundError`` sites keep working. +""" + +from __future__ import annotations + +from typing import Any + + +class ToolError(RuntimeError): + code: str = "tool_error" + + def __init__(self, message: str, *, code: str | None = None, **details: Any) -> None: + super().__init__(message) + if code is not None: + self.code = code + self.details: dict[str, Any] = dict(details) + + +class ToolArgumentError(ToolError): + """A tool call carried a missing, mistyped or out-of-range argument.""" + + code = "invalid_arguments" + + def __init__(self, message: str, *, field: str | None = None, **details: Any) -> None: + super().__init__(message, **details) + self.field = field + + +class WorkspaceBoundaryError(ToolError, PermissionError): + code = "outside_workspace" + + +class ToolFileNotFoundError(ToolError, FileNotFoundError): + code = "file_not_found" + + +class ToolFileExistsError(ToolError, FileExistsError): + code = "file_exists" + + +class NotTextFileError(ToolError): + code = "not_text_file" + + +class FileTooLargeError(ToolError): + code = "file_too_large" + + +class DirectoryNotEmptyError(ToolError): + code = "directory_not_empty" + + +class EditMatchError(ToolError): + """``old_text`` matched zero times (``edit_no_match``) or several (``edit_ambiguous``).""" + + code = "edit_no_match" + + def __init__( + self, + message: str, + *, + code: str, + match_count: int, + edit_index: int = 0, + edit_count: int = 1, + ) -> None: + super().__init__( + message, + code=code, + match_count=match_count, + edit_index=edit_index, + edit_count=edit_count, + ) + self.match_count = match_count + self.edit_index = edit_index + self.edit_count = edit_count + + +class CommandNotFoundError(ToolError): + code = "command_not_found" diff --git a/src/database/crud.py b/src/database/crud.py index 6bc0e24..e67ce71 100644 --- a/src/database/crud.py +++ b/src/database/crud.py @@ -1,9 +1,11 @@ import json import uuid from datetime import datetime +from types import SimpleNamespace from typing import List, Optional, Tuple, Type, Union -from sqlalchemy import func, text +from sqlalchemy import and_, func, or_, select, text +from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session import Queries as Queries @@ -11,12 +13,113 @@ from database.db_schemas import DEFAULT_USER_PREFERENCE from database.embedding_service import encode_text from utils import hash_password, verify_password +from research.canonical import canonical_hash, canonical_json +from database.research_schemas import ( + AgentRelease, + ResearchEnrollment, + ResearchParticipant, +) +from agents.tools import set_harness_profile_fields +from research.study.agents.distributions import validate_profile_configuration +from research.study.agents.enums import QualificationStatus +from research.study.agents.registry import SELECTABLE_STATUSES +from research.study.agents.store import row_to_release + + +class ProfileReleaseError(ValueError): + """Raised when a profile pins a missing or non-selectable release.""" + + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + + +def validate_profile_release(session: Session, release_id: Optional[str]) -> None: + """Validate a supplied profile release without changing registry rows.""" + if release_id is None: + return + release = session.get(AgentRelease, release_id) + if release is None: + raise ProfileReleaseError( + "RELEASE_UNRESOLVED", f"release {release_id!r} is not registered" + ) + status = str(release.status or "").upper() + if status in {QualificationStatus.RETIRED.value, QualificationStatus.BLOCKED.value}: + raise ProfileReleaseError( + "RELEASE_WITHDRAWN", f"release {release_id!r} is withdrawn" + ) + if status not in {item.value for item in SELECTABLE_STATUSES}: + raise ProfileReleaseError( + "RELEASE_NOT_QUALIFIED", f"release {release_id!r} is not qualified" + ) + + +def _validate_profile_configuration( + db: Session, + *, + name: str, + framework_version: str, + release_id: Optional[str], + tools_json: str, + approval_policy: str, + model: Optional[str] = None, + temperature: Optional[float] = None, + max_steps: Optional[int] = None, + max_context_tokens: Optional[int] = None, + system_prompt: Optional[str] = None, + commands_denylist: Optional[list] = None, + command_timeout_seconds: Optional[int] = None, + harness_options: Optional[dict] = None, +) -> None: + """Enforce the shared profile↔release executable contract (ISSUE-03/17). + + The candidate fields are validated together against the rehydrated release + and its stored conformance evidence, so a profile that cannot execute is + rejected at create/update time with a typed reason. ``model`` / + ``temperature`` / ``max_steps`` participate in the BYOA field-coverage + check and ``max_context_tokens`` / ``system_prompt`` and the built-in + runtime's command/harness settings (``commands_denylist`` / + ``command_timeout_seconds`` / ``harness_options``) in the BYOA + unsupported-field check; omitting them would let an ungoverned field + through until study creation. The command/harness settings are also + checked together (a verify command may not run a blocked program). + """ + if release_id is None: + raise ProfileReleaseError( + "RELEASE_UNRESOLVED", "a profile must pin a registered release" + ) + row = db.get(AgentRelease, release_id) + if row is None: + raise ProfileReleaseError( + "RELEASE_UNRESOLVED", f"release {release_id!r} is not registered" + ) + candidate = SimpleNamespace( + name=name, + framework_version=framework_version, + release_id=release_id, + tools_json=tools_json, + approval_policy=approval_policy, + model=model, + temperature=temperature, + max_steps=max_steps, + max_context_tokens=max_context_tokens, + system_prompt=system_prompt, + commands_denylist=commands_denylist, + command_timeout_seconds=command_timeout_seconds, + harness_options=harness_options, + ) + validate_profile_configuration( + candidate, row_to_release(row), release_json=row.release_json + ) + + +def _optional_json_text(value) -> Optional[str]: + """Canonical JSON text for an optional JSON profile column (``None`` stays NULL).""" + return None if value is None else canonical_json(value) # User -def create_user( - db: Session, user: Union[Queries.CreateUser, Queries.CreateUserOauth] -) -> db_schemas.User: +def create_user(db: Session, user: Queries.CreateUser) -> db_schemas.User: # Create user object db_user = db_schemas.User( user_id=uuid.uuid4(), @@ -27,7 +130,6 @@ def create_user( # configure default preference and config upon creation config_id=user.config_id, preference=json.dumps(DEFAULT_USER_PREFERENCE), - is_oauth_signup=isinstance(user, Queries.CreateUserOauth), verified=False, ) @@ -83,6 +185,11 @@ def update_user( def delete_user_by_id(db: Session, user_id: uuid.UUID) -> bool: + """Delete only the account row. + + Not for user-initiated deletion: it leaves collected data behind and cascades + into studies the account owns. Use ``privacy.erasure.delete_account``. + """ result = ( db.query(db_schemas.User).filter(db_schemas.User.user_id == user_id).delete() ) @@ -90,83 +197,6 @@ def delete_user_by_id(db: Session, user_id: uuid.UUID) -> bool: return result > 0 -def delete_user_full_wipe_out(db: Session, user_id: uuid.UUID): - meta_queries = ( - db.query(db_schemas.MetaQuery) - .filter(db_schemas.MetaQuery.user_id == user_id) - .all() - ) - db.query(db_schemas.MetaQuery).filter( - db_schemas.MetaQuery.user_id == user_id - ).delete() - project_users = ( - db.query(db_schemas.ProjectUser) - .filter(db_schemas.ProjectUser.user_id == user_id) - .all() - ) - # TODO: Is not fully tested yet because in current settings the frontend doesn't support multi user project edit - for project_user in project_users: - project_context_should_be_deleted = True - project_should_be_deleted = True - common_project_users = ( - db.query(db_schemas.ProjectUser) - .filter(db_schemas.ProjectUser.project_id == project_user.project_id) - .all() - ) - # Check if there exists another user working on this project don't delete this project as a whole - if len(common_project_users) > 1: - project_should_be_deleted = False - # Check if there exists a user who has agreed to store context on the same project keep the context - for common_project_user in common_project_users: - common_user = ( - db.query(db_schemas.User) - .filter(db_schemas.User.user_id == common_project_user.user_id) - .first() - ) - if common_user and json.loads(common_user.preference).get( - "store_context", False - ): - project_context_should_be_deleted = False - break - if project_should_be_deleted: - db.query(db_schemas.Project).filter( - db_schemas.Project.project_id == project_user.project_id - ).delete() - elif project_context_should_be_deleted: - db.query(db_schemas.Project).filter( - db_schemas.Project.project_id == project_user.project_id - ).update({"multi_file_contexts": "{}", "multi_file_context_changes": "{}"}) - - db.query(db_schemas.ProjectUser).filter( - db_schemas.ProjectUser.user_id == user_id - ).delete() - db.query(db_schemas.Context).filter( - db_schemas.Context.context_id.in_( - list(map(lambda x: x.context_id, meta_queries)) - ) - ).delete() - db.query(db_schemas.BehavioralTelemetry).filter( - db_schemas.BehavioralTelemetry.behavioral_telemetry_id.in_( - list(map(lambda x: x.behavioral_telemetry_id, meta_queries)) - ) - ).delete() - db.query(db_schemas.ContextualTelemetry).filter( - db_schemas.ContextualTelemetry.contextual_telemetry_id.in_( - list(map(lambda x: x.contextual_telemetry_id, meta_queries)) - ) - ).delete() - - db.query(db_schemas.Session).filter(db_schemas.Session.user_id == user_id).delete() - db.query(db_schemas.Chat).filter(db_schemas.Chat.user_id == user_id).delete() - db.query(db_schemas.HadGeneration).filter( - db_schemas.HadGeneration.meta_query_id.in_( - list(map(lambda x: x.meta_query_id, meta_queries)) - ) - ).delete() - db.query(db_schemas.User).filter(db_schemas.User.user_id == user_id).delete() - db.commit() - - # Context Operations def create_context( db: Session, context: Queries.ContextData, context_id: str = "" @@ -229,27 +259,6 @@ def create_behavioral_telemetry( return db_telemetry -def get_contextual_telemetry_by_id( - db: Session, telemetry_id: uuid.UUID -) -> Optional[db_schemas.ContextualTelemetry]: - return ( - db.query(db_schemas.ContextualTelemetry) - .filter(db_schemas.ContextualTelemetry.contextual_telemetry_id == telemetry_id) - .first() - ) - - -def get_behavioral_telemetry_by_id( - db: Session, telemetry_id: uuid.UUID -) -> Optional[db_schemas.BehavioralTelemetry]: - return ( - db.query(db_schemas.BehavioralTelemetry) - .filter(db_schemas.BehavioralTelemetry.behavioral_telemetry_id == telemetry_id) - .first() - ) - - -# MetaQuery operations def create_completion_query( db: Session, query: Queries.CreateCompletionQuery, id: str = "" ) -> db_schemas.CompletionQuery: @@ -336,26 +345,6 @@ def get_meta_query_by_id( ) -def get_completion_query_by_id( - db: Session, meta_query_id: uuid.UUID -) -> Optional[db_schemas.CompletionQuery]: - return ( - db.query(db_schemas.CompletionQuery) - .filter(db_schemas.CompletionQuery.meta_query_id == meta_query_id) - .first() - ) - - -def get_chat_query_by_id( - db: Session, meta_query_id: uuid.UUID -) -> Optional[db_schemas.ChatQuery]: - return ( - db.query(db_schemas.ChatQuery) - .filter(db_schemas.ChatQuery.meta_query_id == meta_query_id) - .first() - ) - - def get_chat_queries_for_chat( db: Session, chat_id: uuid.UUID ) -> list[db_schemas.ChatQuery]: @@ -405,16 +394,6 @@ def create_generation( return db_generation -def get_generations_by_meta_query( - db: Session, meta_query_id: uuid.UUID -) -> list[db_schemas.HadGeneration]: - return ( - db.query(db_schemas.HadGeneration) - .filter(db_schemas.HadGeneration.meta_query_id == meta_query_id) - .all() - ) - - # def update_generation_acceptance( # db: Session, update_data: Queries.UpdateGenerationAcceptance # ) -> Optional[db_schemas.HadGeneration]: @@ -904,7 +883,24 @@ def delete_session_cascade(db: Session, session_id: uuid.UUID) -> bool: def create_ground_truth( db: Session, ground_truth: Queries.CreateGroundTruth ) -> db_schemas.GroundTruth: - """Create a ground truth record""" + """Create a ground truth record. + + Idempotent for retried feedback submissions: an identical + (completion_query_id, ground_truth) snapshot is stored only once, so a + retried POST never double-saves the same collected telemetry. Distinct + snapshots (e.g. the 15/45/90s captures) are still stored as separate rows. + """ + existing = ( + db.query(db_schemas.GroundTruth) + .filter( + db_schemas.GroundTruth.completion_query_id + == ground_truth.completion_query_id, + db_schemas.GroundTruth.ground_truth == ground_truth.ground_truth, + ) + .first() + ) + if existing is not None: + return existing db_ground_truth = db_schemas.GroundTruth( completion_query_id=ground_truth.completion_query_id, truth_timestamp=datetime.now(), @@ -916,16 +912,6 @@ def create_ground_truth( return db_ground_truth -def get_ground_truths_for_completion( - db: Session, completion_query_id: uuid.UUID -) -> list[db_schemas.GroundTruth]: - return ( - db.query(db_schemas.GroundTruth) - .filter(db_schemas.GroundTruth.completion_query_id == completion_query_id) - .all() - ) - - def get_all_programming_languages(db: Session) -> list[db_schemas.ProgrammingLanguage]: return db.query(db_schemas.ProgrammingLanguage).all() @@ -1124,37 +1110,6 @@ def search_similar_documentation( return [] -def regenerate_embeddings(db: Session, language: Optional[str] = None) -> int: - """ - Regenerate embeddings for all documentation entries. - Useful when changing embedding models or fixing corrupted embeddings. - - Args: - language: Optional language filter to regenerate only specific language docs - - Returns: - Number of embeddings regenerated - """ - query = db.query(db_schemas.Documentation) - - if language: - query = query.filter(db_schemas.Documentation.language == language) - - docs = query.all() - - updated_count = 0 - for doc in docs: - try: - new_embedding = encode_text(doc.content) - doc.embedding = new_embedding - updated_count += 1 - except Exception as e: - print(f"Failed to regenerate embedding for doc {doc.documentation_id}: {e}") - - db.commit() - return updated_count - - def get_documentation_stats(db: Session) -> dict: """Get statistics about documentation entries.""" total_docs = db.query(db_schemas.Documentation).count() @@ -1183,3 +1138,1136 @@ def get_documentation_stats(db: Session) -> dict: ), "languages": dict(language_stats), } + + +# ── Agent CRUD ──────────────────────────────────────────────────────────────── +# +# Content parameters (task_description, first_system_message, last_user_message, +# response_text, tool_arguments, tool_result, payload_json, diff_text, +# edit_delta_json) are optional and default to None. Callers are responsible for +# passing None unless the user's store_agent_content preference resolves True — +# see backend.routers.agent.consent.resolve_store_agent_content. These helpers +# deliberately do not re-check consent, so there is exactly one place where that +# decision is made. + + +class ProfileLockedError(PermissionError): + """Raised when an active research study owns a profile selection.""" + + +def _agent_profile_configuration(profile: db_schemas.AgentProfile) -> dict: + configuration = { + "profile_id": str(profile.profile_id), + "name": profile.name, + "model": profile.model, + "framework_version": profile.framework_version, + "connection_id": str(profile.connection_id) if profile.connection_id else None, + "release_id": profile.release_id, + "tools_json": profile.tools_json, + "approval_policy": profile.approval_policy, + "max_steps": profile.max_steps, + "temperature": profile.temperature, + "max_context_tokens": profile.max_context_tokens, + "is_active": profile.is_active, + } + # Only a set prompt joins the digest input, so every profile without one + # keeps the byte-identical configuration digest it had before the column. + system_prompt = getattr(profile, "system_prompt", None) + if system_prompt is not None: + configuration["system_prompt"] = system_prompt + # Likewise the built-in runtime's command/harness settings (D-01): only set + # values join, so a profile without them keeps its digest. + configuration.update(set_harness_profile_fields(profile)) + return configuration + + +def _refresh_agent_profile_digest(profile: db_schemas.AgentProfile) -> None: + profile.configuration_digest = canonical_hash(_agent_profile_configuration(profile)) + + +def _assert_agent_profile_editable(db: Session, profile_id: uuid.UUID) -> None: + from database.research_schemas import StudyAgentProfile + + linked_active = ( + db.query(StudyAgentProfile.study_id) + .join( + db_schemas.Study, + StudyAgentProfile.study_id == db_schemas.Study.study_id, + ) + .filter( + StudyAgentProfile.profile_id == profile_id, + db_schemas.Study.is_research.is_(True), + db_schemas.Study.research_status == "ACTIVE", + ) + .first() + ) + if linked_active is not None: + raise ProfileLockedError( + "profile is locked while linked research study is ACTIVE" + ) + + +def create_agent_profile( + db: Session, + *, + owner_user_id: uuid.UUID, + name: str, + model: str, + tools_json: str, + approval_policy: str, + max_steps: int, + framework_version: str = "code4me2-agent", + connection_id: Optional[uuid.UUID] = None, + release_id: Optional[str] = None, + is_active: bool = True, + max_context_tokens: Optional[int] = None, + temperature: Optional[float] = None, + system_prompt: Optional[str] = None, + commands_denylist: Optional[list] = None, + command_timeout_seconds: Optional[int] = None, + harness_options: Optional[dict] = None, +) -> db_schemas.AgentProfile: + """Create a researcher-owned profile template. + + The provider endpoint/secret live on the referenced ``provider_connection``; + a profile never stores a URL or a secret reference. ``commands_denylist`` + and ``harness_options`` are stored as canonical JSON text. + """ + validate_profile_release(db, release_id) + _validate_profile_configuration( + db, + name=name, + framework_version=framework_version, + release_id=release_id, + tools_json=tools_json, + approval_policy=approval_policy, + model=model, + temperature=temperature, + max_steps=max_steps, + max_context_tokens=max_context_tokens, + system_prompt=system_prompt, + commands_denylist=commands_denylist, + command_timeout_seconds=command_timeout_seconds, + harness_options=harness_options, + ) + profile = db_schemas.AgentProfile( + profile_id=uuid.uuid4(), + owner_user_id=owner_user_id, + name=name, + model=model, + framework_version=framework_version, + connection_id=connection_id, + release_id=release_id, + tools_json=tools_json, + approval_policy=approval_policy, + max_steps=max_steps, + is_active=is_active, + max_context_tokens=max_context_tokens, + temperature=temperature, + system_prompt=system_prompt, + commands_denylist_json=_optional_json_text(commands_denylist), + command_timeout_seconds=command_timeout_seconds, + harness_options_json=_optional_json_text(harness_options), + ) + _refresh_agent_profile_digest(profile) + db.add(profile) + db.commit() + db.refresh(profile) + return profile + + +def get_agent_profile_by_id( + db: Session, profile_id: uuid.UUID +) -> Optional[db_schemas.AgentProfile]: + return ( + db.query(db_schemas.AgentProfile) + .filter(db_schemas.AgentProfile.profile_id == profile_id) + .first() + ) + + +def list_agent_profiles( + db: Session, owner_user_id: Optional[uuid.UUID] = None +) -> List[db_schemas.AgentProfile]: + """List profiles; ``owner_user_id`` scopes to one researcher (admin = all).""" + query = db.query(db_schemas.AgentProfile) + if owner_user_id is not None: + query = query.filter(db_schemas.AgentProfile.owner_user_id == owner_user_id) + return query.order_by(db_schemas.AgentProfile.name).all() + + +def update_agent_profile( + db: Session, + profile_id: uuid.UUID, + *, + name: Optional[str] = None, + model: Optional[str] = None, + tools_json: Optional[str] = None, + approval_policy: Optional[str] = None, + max_steps: Optional[int] = None, + framework_version: Optional[str] = None, + connection_id: Optional[uuid.UUID] = None, + release_id: Optional[str] = None, + is_active: Optional[bool] = None, + max_context_tokens: Optional[int] = None, + temperature: Optional[float] = None, + system_prompt: Optional[str] = None, + commands_denylist: Optional[list] = None, + command_timeout_seconds: Optional[int] = None, + harness_options: Optional[dict] = None, + update_connection_id: bool = False, + update_release_id: bool = False, + update_temperature: bool = False, + update_max_context_tokens: bool = False, + update_system_prompt: bool = False, + update_commands_denylist: bool = False, + update_command_timeout_seconds: bool = False, + update_harness_options: bool = False, +) -> Optional[db_schemas.AgentProfile]: + """Update a profile template in place (caller performs ownership checks). + + ``None`` keeps a field unchanged, except where the matching ``update_*`` + flag is set: then the supplied value is stored as-is, so ``None`` clears the + optional ``temperature`` / ``max_context_tokens`` / ``system_prompt`` / + ``commands_denylist`` / ``command_timeout_seconds`` / ``harness_options`` + overrides (a full-replace PUT). The refreshed digest is then exactly that of + a profile created with ``NULL`` for the cleared field. + """ + profile = ( + db.query(db_schemas.AgentProfile) + .filter(db_schemas.AgentProfile.profile_id == profile_id) + .with_for_update() + .first() + ) + if profile is None: + return None + _assert_agent_profile_editable(db, profile_id) + if update_release_id: + validate_profile_release(db, release_id) + effective_temperature = ( + temperature + if update_temperature or temperature is not None + else getattr(profile, "temperature", None) + ) + effective_max_context_tokens = ( + max_context_tokens + if update_max_context_tokens or max_context_tokens is not None + else getattr(profile, "max_context_tokens", None) + ) + effective_system_prompt = ( + system_prompt + if update_system_prompt or system_prompt is not None + else getattr(profile, "system_prompt", None) + ) + effective_commands_denylist = ( + commands_denylist + if update_commands_denylist or commands_denylist is not None + else getattr(profile, "commands_denylist", None) + ) + effective_command_timeout_seconds = ( + command_timeout_seconds + if update_command_timeout_seconds or command_timeout_seconds is not None + else getattr(profile, "command_timeout_seconds", None) + ) + effective_harness_options = ( + harness_options + if update_harness_options or harness_options is not None + else getattr(profile, "harness_options", None) + ) + # Validate the merged result, not just the supplied fields: changing only + # the framework (or only the release) must not leave an unexecutable pair. + _validate_profile_configuration( + db, + name=name if name is not None else profile.name, + framework_version=( + framework_version if framework_version is not None else profile.framework_version + ), + release_id=release_id if update_release_id else profile.release_id, + tools_json=tools_json if tools_json is not None else profile.tools_json, + approval_policy=( + approval_policy if approval_policy is not None else profile.approval_policy + ), + model=model if model is not None else profile.model, + temperature=effective_temperature, + max_steps=max_steps if max_steps is not None else profile.max_steps, + max_context_tokens=effective_max_context_tokens, + system_prompt=effective_system_prompt, + commands_denylist=effective_commands_denylist, + command_timeout_seconds=effective_command_timeout_seconds, + harness_options=effective_harness_options, + ) + if name is not None: + profile.name = name + if model is not None: + profile.model = model + if framework_version is not None: + profile.framework_version = framework_version + if tools_json is not None: + profile.tools_json = tools_json + if approval_policy is not None: + profile.approval_policy = approval_policy + if max_steps is not None: + profile.max_steps = max_steps + if update_connection_id: + profile.connection_id = connection_id + if update_release_id: + profile.release_id = release_id + if is_active is not None: + profile.is_active = is_active + if update_max_context_tokens or max_context_tokens is not None: + profile.max_context_tokens = max_context_tokens + if update_temperature or temperature is not None: + profile.temperature = temperature + if update_system_prompt or system_prompt is not None: + profile.system_prompt = system_prompt + if update_commands_denylist or commands_denylist is not None: + profile.commands_denylist_json = _optional_json_text(commands_denylist) + if update_command_timeout_seconds or command_timeout_seconds is not None: + profile.command_timeout_seconds = command_timeout_seconds + if update_harness_options or harness_options is not None: + profile.harness_options_json = _optional_json_text(harness_options) + _refresh_agent_profile_digest(profile) + db.commit() + db.refresh(profile) + return profile + + +def delete_agent_profile(db: Session, profile_id: uuid.UUID) -> bool: + """Archive a profile template (never erase a frozen study snapshot).""" + profile = ( + db.query(db_schemas.AgentProfile) + .filter(db_schemas.AgentProfile.profile_id == profile_id) + .with_for_update() + .first() + ) + if profile is None: + return False + _assert_agent_profile_editable(db, profile_id) + if not profile.is_active: + return True + profile.is_active = False + db.commit() + return True + + +# ── Admin-managed provider connections (role + readiness) ────────────────── + + +def create_provider_connection( + db: Session, + *, + label: str, + base_url: str, + secret_ref: str, + models_json: str, + is_active: bool = True, +) -> db_schemas.ProviderConnection: + connection = db_schemas.ProviderConnection( + connection_id=uuid.uuid4(), + label=label, + base_url=base_url, + secret_ref=secret_ref, + models_json=models_json, + is_active=is_active, + ) + db.add(connection) + db.commit() + db.refresh(connection) + return connection + + +def get_provider_connection( + db: Session, connection_id: uuid.UUID +) -> Optional[db_schemas.ProviderConnection]: + return db.get(db_schemas.ProviderConnection, connection_id) + + +def get_provider_connection_by_label( + db: Session, label: str +) -> Optional[db_schemas.ProviderConnection]: + return ( + db.query(db_schemas.ProviderConnection) + .filter(db_schemas.ProviderConnection.label == label) + .first() + ) + + +def list_provider_connections(db: Session) -> List[db_schemas.ProviderConnection]: + return ( + db.query(db_schemas.ProviderConnection) + .order_by(db_schemas.ProviderConnection.label) + .all() + ) + + +def update_provider_connection( + db: Session, + connection_id: uuid.UUID, + *, + label: Optional[str] = None, + base_url: Optional[str] = None, + secret_ref: Optional[str] = None, + models_json: Optional[str] = None, + is_active: Optional[bool] = None, +) -> Optional[db_schemas.ProviderConnection]: + connection = db.get(db_schemas.ProviderConnection, connection_id) + if connection is None: + return None + if label is not None: + connection.label = label + if base_url is not None: + connection.base_url = base_url + if secret_ref is not None: + connection.secret_ref = secret_ref + if models_json is not None: + connection.models_json = models_json + if is_active is not None: + connection.is_active = is_active + db.commit() + db.refresh(connection) + return connection + + +def delete_provider_connection(db: Session, connection_id: uuid.UUID) -> bool: + connection = db.get(db_schemas.ProviderConnection, connection_id) + if connection is None: + return False + db.delete(connection) + db.commit() + return True + + +def list_model_prices( + db: Session, connection_id: uuid.UUID +) -> List[db_schemas.ProviderModelPrice]: + """Price rows of one connection, ordered by model name.""" + return ( + db.query(db_schemas.ProviderModelPrice) + .filter(db_schemas.ProviderModelPrice.connection_id == connection_id) + .order_by(db_schemas.ProviderModelPrice.model) + .all() + ) + + +def list_model_prices_by_connection( + db: Session, +) -> dict: + """``{connection_id: {model: price_row}}`` for every connection.""" + result: dict = {} + for row in db.query(db_schemas.ProviderModelPrice).all(): + result.setdefault(row.connection_id, {})[row.model] = row + return result + + +def replace_model_prices( + db: Session, + connection_id: uuid.UUID, + prices: dict, + *, + allowed_models: Optional[List[str]] = None, + updated_by: Optional[str] = None, +) -> List[db_schemas.ProviderModelPrice]: + """Upsert the given ``{model: {input, output, cached_input}}`` prices. + + A ``None`` value deletes that model's price. Price rows for models no + longer in ``allowed_models`` are deleted too, so a renamed model never + keeps a stale price. Commits. + """ + from datetime import datetime, timezone + + now = datetime.now(timezone.utc) + existing = {row.model: row for row in list_model_prices(db, connection_id)} + for model, price in prices.items(): + row = existing.get(model) + if price is None: + if row is not None: + db.delete(row) + continue + if row is None: + row = db_schemas.ProviderModelPrice(connection_id=connection_id, model=model) + db.add(row) + row.input_usd_per_million = price["input_usd_per_million"] + row.output_usd_per_million = price["output_usd_per_million"] + row.cached_input_usd_per_million = price.get("cached_input_usd_per_million") + row.updated_at = now + row.updated_by = updated_by + if allowed_models is not None: + allowed = set(allowed_models) + for model, row in existing.items(): + if model not in allowed and model not in prices: + db.delete(row) + db.commit() + return list_model_prices(db, connection_id) + + +def count_profiles_for_connection(db: Session, connection_id: uuid.UUID) -> int: + """Agent profiles (active or archived) that reference one connection. + + Every referencing row blocks deletion: ``agent_profile.connection_id`` is a + ``RESTRICT`` foreign key, and archived profiles keep their reference. + """ + return int( + db.query(func.count(db_schemas.AgentProfile.profile_id)) + .filter(db_schemas.AgentProfile.connection_id == connection_id) + .scalar() + or 0 + ) + + +def count_profiles_by_connection(db: Session) -> dict: + """``{connection_id: referencing profile count}`` in one grouped query.""" + rows = ( + db.query( + db_schemas.AgentProfile.connection_id, + func.count(db_schemas.AgentProfile.profile_id), + ) + .filter(db_schemas.AgentProfile.connection_id.isnot(None)) + .group_by(db_schemas.AgentProfile.connection_id) + .all() + ) + return {connection_id: int(count) for connection_id, count in rows} + + +def list_available_provider_connections( + db: Session, user_id: uuid.UUID +) -> List[db_schemas.ProviderConnection]: + """Active admin-managed connections available to an authorized researcher.""" + return ( + db.query(db_schemas.ProviderConnection) + .filter(db_schemas.ProviderConnection.is_active.is_(True)) + .order_by(db_schemas.ProviderConnection.label) + .all() + ) + + +def provider_connection_is_available( + db: Session, connection_id: uuid.UUID, user_id: uuid.UUID +) -> bool: + """Whether an active admin-managed connection can be selected.""" + return ( + db.query(db_schemas.ProviderConnection) + .filter( + db_schemas.ProviderConnection.connection_id == connection_id, + db_schemas.ProviderConnection.is_active.is_(True), + ) + .first() + is not None + ) + + +# ── Researcher enablement ──────────────────────────────────────────────────── + + +def set_user_can_research( + db: Session, user_id: uuid.UUID, enabled: bool +) -> Optional[db_schemas.User]: + """Administrator toggle of the single researcher-enablement flag.""" + user = db.get(db_schemas.User, user_id) + if user is None: + return None + user.can_research = enabled + db.commit() + db.refresh(user) + return user + + +#: Account role filters of the administrator account view. +ACCOUNT_ROLE_FILTERS = ("all", "admin", "researcher", "participant") +#: Account enrollment filters of the administrator account view. +ACCOUNT_ENROLLMENT_FILTERS = ("any", "enrolled", "not_enrolled") + + +def _account_enrollment_exists(*conditions): + """Correlated ``EXISTS`` over the account's enrollments (via its mapping).""" + return ( + select(ResearchEnrollment.enrollment_id) + .join( + ResearchParticipant, + ResearchParticipant.participant_id == ResearchEnrollment.participant_id, + ) + .where(ResearchParticipant.account_id == db_schemas.User.user_id, *conditions) + .exists() + ) + + +def _account_filter_criteria( + *, + q: Optional[str] = None, + role: str = "all", + enrollment: str = "any", + study_id: Optional[uuid.UUID] = None, +) -> list: + """SQL criteria for the administrator account filters. + + * ``q`` — case-insensitive substring over email and name (``%``/``_`` are + matched literally); + * ``role`` — ``admin``; ``researcher`` (``can_research`` and not admin); + ``participant`` (neither); ``all``; + * ``enrollment`` — ``enrolled`` (has an ACTIVE enrollment), + ``not_enrolled`` (has none), ``any``; + * ``study_id`` — has any enrollment (any status) in that study. + """ + if role not in ACCOUNT_ROLE_FILTERS: + raise ValueError(f"unknown account role filter {role!r}") + if enrollment not in ACCOUNT_ENROLLMENT_FILTERS: + raise ValueError(f"unknown account enrollment filter {enrollment!r}") + user = db_schemas.User + criteria = [] + needle = (q or "").strip() + if needle: + criteria.append( + or_( + user.email.icontains(needle, autoescape=True), + user.name.icontains(needle, autoescape=True), + ) + ) + is_admin = func.coalesce(user.is_admin, False) + can_research = func.coalesce(user.can_research, False) + if role == "admin": + criteria.append(is_admin.is_(True)) + elif role == "researcher": + criteria.append(and_(can_research.is_(True), is_admin.is_(False))) + elif role == "participant": + criteria.append(and_(can_research.is_(False), is_admin.is_(False))) + if enrollment == "enrolled": + criteria.append(_account_enrollment_exists(ResearchEnrollment.status == "ACTIVE")) + elif enrollment == "not_enrolled": + criteria.append(~_account_enrollment_exists(ResearchEnrollment.status == "ACTIVE")) + if study_id is not None: + criteria.append(_account_enrollment_exists(ResearchEnrollment.study_id == study_id)) + return criteria + + +def list_accounts( + db: Session, + limit: int = 100, + *, + offset: int = 0, + q: Optional[str] = None, + role: str = "all", + enrollment: str = "any", + study_id: Optional[uuid.UUID] = None, +) -> List[db_schemas.User]: + """One page of accounts, newest first, for the administrator account view. + + The admin panel toggles ``can_research`` per account, so it needs the full + account list rather than only the already-enabled researchers. The optional + filters are :func:`_account_filter_criteria`; ``user_id`` breaks + ``joined_at`` ties so paging is stable. + """ + criteria = _account_filter_criteria( + q=q, role=role, enrollment=enrollment, study_id=study_id + ) + return ( + db.query(db_schemas.User) + .filter(*criteria) + .order_by(db_schemas.User.joined_at.desc(), db_schemas.User.user_id.asc()) + .offset(offset) + .limit(limit) + .all() + ) + + +def count_accounts( + db: Session, + *, + q: Optional[str] = None, + role: str = "all", + enrollment: str = "any", + study_id: Optional[uuid.UUID] = None, +) -> int: + """How many accounts match the filters (before paging).""" + criteria = _account_filter_criteria( + q=q, role=role, enrollment=enrollment, study_id=study_id + ) + return int( + db.query(func.count(db_schemas.User.user_id)).filter(*criteria).scalar() or 0 + ) + + +def list_account_enrollments(db: Session, account_ids) -> dict: + """``{account_id: [enrollment rows, newest first]}`` in one query. + + Each row carries the enrollment id/study/status/``enrolled_at`` plus the + study's name and ``research_status`` (``None`` when the study row is gone). + The participant code and the assigned profile are deliberately not + selected: this administrator view must not link an account to its + study-local pseudonym or arm. + """ + ids = list(account_ids) + grouped: dict = {account_id: [] for account_id in ids} + if not ids: + return grouped + statement = ( + select( + ResearchParticipant.account_id.label("account_id"), + ResearchEnrollment.enrollment_id.label("enrollment_id"), + ResearchEnrollment.study_id.label("study_id"), + ResearchEnrollment.status.label("status"), + ResearchEnrollment.enrolled_at.label("enrolled_at"), + db_schemas.Study.name.label("study_name"), + db_schemas.Study.research_status.label("study_status"), + ) + .select_from(ResearchEnrollment) + .join( + ResearchParticipant, + ResearchParticipant.participant_id == ResearchEnrollment.participant_id, + ) + .outerjoin( + db_schemas.Study, + db_schemas.Study.study_id == ResearchEnrollment.study_id, + ) + .where(ResearchParticipant.account_id.in_(ids)) + .order_by( + ResearchEnrollment.enrolled_at.desc(), + ResearchEnrollment.enrollment_id.asc(), + ) + ) + for row in db.execute(statement).all(): + grouped.setdefault(row.account_id, []).append(row) + return grouped + + +# ── Agent tasks ───────────────────────────────────────────────────────────── + + +def create_agent_task( + db: Session, + agent_profile: str, + model: str, + approval_policy: str, + tools_json: str, + task_description: Optional[str] = None, + session_id: Optional[uuid.UUID] = None, + task_id: Optional[uuid.UUID] = None, + temperature: Optional[float] = None, + framework_version: Optional[str] = None, + source: str = "plugin", + owner_user_id: Optional[uuid.UUID] = None, + funding_owner_user_id: Optional[uuid.UUID] = None, + owner_project_id: Optional[uuid.UUID] = None, + external_run_id: Optional[str] = None, + agent_session_id: Optional[str] = None, + status: str = "pending", + started_at: Optional[datetime] = None, + policy_snapshot: Optional[dict] = None, + study_id: Optional[uuid.UUID] = None, + study_assignment_id: Optional[uuid.UUID] = None, + profile_id: Optional[uuid.UUID] = None, + consent_content_storage: Optional[bool] = None, + research_session_id: Optional[uuid.UUID] = None, + enrollment_id: Optional[uuid.UUID] = None, +) -> db_schemas.AgentTask: + # Dashboards join research_event.agent_run_id to external_run_id, so a task + # without a runtime-supplied run id still needs a stable correlation id. + # Otherwise its canonical events are orphaned and invisible to researchers. + # Callers with a real external run (ACP/managed/proxy flows) pass it + # explicitly and it is preserved; uuid4 can never collide with those ids. + task = db_schemas.AgentTask( + task_id=task_id or uuid.uuid4(), + agent_profile=agent_profile, + model=model, + temperature=temperature, + approval_policy=approval_policy, + tools_json=tools_json, + framework_version=framework_version, + status=status, + source=source, + session_id=session_id, + owner_user_id=owner_user_id, + funding_owner_user_id=funding_owner_user_id, + owner_project_id=owner_project_id, + external_run_id=external_run_id or uuid.uuid4().hex, + agent_session_id=agent_session_id, + study_id=study_id, + study_assignment_id=study_assignment_id, + profile_id=profile_id, + consent_content_storage=consent_content_storage, + research_session_id=research_session_id, + enrollment_id=enrollment_id, + started_at=started_at, + policy_snapshot=policy_snapshot, + task_description=task_description, + ) + db.add(task) + db.commit() + db.refresh(task) + return task + + +def get_agent_task(db: Session, task_id: uuid.UUID) -> Optional[db_schemas.AgentTask]: + return ( + db.query(db_schemas.AgentTask) + .filter(db_schemas.AgentTask.task_id == task_id) + .first() + ) + + +def get_agent_task_by_external_run_id( + db: Session, external_run_id: str +) -> Optional[db_schemas.AgentTask]: + """Look a task up by the *runtime's* own run id. + + The self-reporting ``code4me2-agent`` runtime mints its own run id before it + ever talks to the backend, so ingestion resolves the task this way rather + than requiring the runtime to adopt a server-issued UUID. + """ + return ( + db.query(db_schemas.AgentTask) + .filter(db_schemas.AgentTask.external_run_id == external_run_id) + .first() + ) + + +def get_open_agent_tasks_for_session( + db: Session, session_id: uuid.UUID +) -> List[db_schemas.AgentTask]: + """Tasks belonging to this session that haven't reached a terminal state yet.""" + return ( + db.query(db_schemas.AgentTask) + .filter( + db_schemas.AgentTask.session_id == session_id, + db_schemas.AgentTask.status.notin_(["done", "failed"]), + ) + .all() + ) + + +def update_agent_task_status( + db: Session, + task_id: uuid.UUID, + status: str, + completed_at: Optional[datetime] = None, + total_steps: Optional[int] = None, + input_tokens: Optional[int] = None, + output_tokens: Optional[int] = None, + latest_event_id: Optional[uuid.UUID] = None, +) -> bool: + data: dict = {"status": status} + if completed_at is not None: + data["completed_at"] = completed_at + if total_steps is not None: + data["total_steps"] = total_steps + if input_tokens is not None: + data["input_tokens"] = input_tokens + if output_tokens is not None: + data["output_tokens"] = output_tokens + if latest_event_id is not None: + data["latest_event_id"] = latest_event_id + result = ( + db.query(db_schemas.AgentTask) + .filter(db_schemas.AgentTask.task_id == task_id) + .update(data) + ) + db.commit() + return result > 0 + + +def update_agent_task_tools( + db: Session, + task_id: uuid.UUID, + tools: List[str], +) -> None: + db.query(db_schemas.AgentTask).filter( + db_schemas.AgentTask.task_id == task_id + ).update({"observed_tools_json": json.dumps(tools)}) + db.commit() + + +def set_agent_task_description( + db: Session, task_id: uuid.UUID, description: str +) -> None: + db.query(db_schemas.AgentTask).filter( + db_schemas.AgentTask.task_id == task_id + ).update({"task_description": description}) + db.commit() + + +def set_agent_task_framework_version( + db: Session, task_id: uuid.UUID, framework_version: str +) -> None: + db.query(db_schemas.AgentTask).filter( + db_schemas.AgentTask.task_id == task_id + ).update({"observed_framework_version": framework_version}) + db.commit() + + +# ── Agent events ──────────────────────────────────────────────────────────── + + +def get_agent_events_by_task( + db: Session, + task_id: uuid.UUID, + event_type: Optional[str] = None, +) -> List[db_schemas.AgentEvent]: + q = db.query(db_schemas.AgentEvent).filter(db_schemas.AgentEvent.task_id == task_id) + if event_type is not None: + q = q.filter(db_schemas.AgentEvent.event_type == event_type) + return q.order_by(db_schemas.AgentEvent.event_index).all() + + +def get_last_agent_event_for_session( + db: Session, + session_id: uuid.UUID, + event_type: Optional[str] = None, +) -> Optional[db_schemas.AgentEvent]: + """Most recent event across all tasks belonging to this session — used to detect + chat-session boundaries that span idle-close task gaps.""" + q = ( + db.query(db_schemas.AgentEvent) + .join( + db_schemas.AgentTask, + db_schemas.AgentEvent.task_id == db_schemas.AgentTask.task_id, + ) + .filter(db_schemas.AgentTask.session_id == session_id) + ) + if event_type is not None: + q = q.filter(db_schemas.AgentEvent.event_type == event_type) + return q.order_by(db_schemas.AgentEvent.created_at.desc()).first() + + +def reserve_agent_event_indexes( + db: Session, task_id: uuid.UUID, event_count: int +) -> int: + """Atomically reserve ``event_count`` task-local event indexes.""" + if event_count < 1: + raise ValueError("event_count must be positive") + first_index = db.execute( + text( + """ + UPDATE public.agent_task + SET next_event_index = next_event_index + :event_count + WHERE task_id = :task_id + RETURNING next_event_index - :event_count + """ + ), + {"event_count": event_count, "task_id": task_id}, + ).scalar_one_or_none() + if first_index is None: + raise ValueError(f"Agent task {task_id} does not exist") + return int(first_index) + + +def find_existing_agent_event_source_ids( + db: Session, + *, + task_id: uuid.UUID, + source: str, + source_event_ids: List[str], +) -> List[str]: + """Return source event ids already persisted for this task and source. + + This lookup is advisory only. The matching unique constraint is the + authoritative protection against concurrent retries. + """ + if not source_event_ids: + return [] + return [ + row.source_event_id + for row in db.query(db_schemas.AgentEvent.source_event_id) + .filter( + db_schemas.AgentEvent.task_id == task_id, + db_schemas.AgentEvent.source == source, + db_schemas.AgentEvent.source_event_id.in_(source_event_ids), + ) + .all() + ] + + +def append_agent_event( + db: Session, + task_id: uuid.UUID, + event_index: int, + event_type: str, + latency_ms: Optional[int] = None, + source: Optional[str] = None, + source_event_id: Optional[str] = None, + schema_version: Optional[str] = None, + occurred_at: Optional[datetime] = None, + # Shared span identifiers (model_call and tool_call) + span_id: Optional[str] = None, + parent_span_id: Optional[uuid.UUID] = None, + request_id: Optional[str] = None, + chat_session_index: Optional[int] = None, + # model_call fields + model: Optional[str] = None, + agent_profile: Optional[str] = None, + streaming: Optional[bool] = None, + message_count: Optional[int] = None, + role_system_count: Optional[int] = None, + role_user_count: Optional[int] = None, + role_assistant_count: Optional[int] = None, + role_tool_count: Optional[int] = None, + tools_kept: Optional[int] = None, + tools_stripped: Optional[int] = None, + tool_names_requested: Optional[list] = None, + max_tokens: Optional[int] = None, + prompt_tokens: Optional[int] = None, + completion_tokens: Optional[int] = None, + total_tokens: Optional[int] = None, + finish_reason: Optional[str] = None, + upstream_status: Optional[int] = None, + step_index: Optional[int] = None, + context_window_size_bytes: Optional[int] = None, + active_file: Optional[str] = None, + first_message_hash: Optional[str] = None, + chat_new_session_detected: Optional[bool] = None, + experiment_tool_access_enabled: Optional[bool] = None, + experiment_approval_policy: Optional[str] = None, + # model_call content (only when store_agent_content resolves True) + first_system_message: Optional[str] = None, + last_user_message: Optional[str] = None, + response_text: Optional[str] = None, + # tool_call fields + tool_name: Optional[str] = None, + tool_arguments_length: Optional[int] = None, + tool_result_length: Optional[int] = None, + # tool_call content (only when store_agent_content resolves True) + tool_arguments: Optional[str] = None, + tool_result: Optional[str] = None, + # JSON overflow + extra_json: Optional[str] = None, + # content (only when store_agent_content resolves True) + payload_json: Optional[str] = None, + ignore_duplicate_source: bool = False, + commit: bool = True, +) -> Optional[db_schemas.AgentEvent]: + """Append one event row. + + ``commit=False`` lets a caller stage a whole batch and commit once (used by + the self-report ingestion path, where a partially-written batch would leave + gaps in ``event_index``). + """ + event = db_schemas.AgentEvent( + event_id=uuid.uuid4(), + task_id=task_id, + event_index=event_index, + event_type=event_type, + source=source, + source_event_id=source_event_id, + schema_version=schema_version, + latency_ms=latency_ms, + occurred_at=occurred_at, + span_id=span_id, + parent_span_id=parent_span_id, + request_id=request_id, + chat_session_index=chat_session_index, + model=model, + agent_profile=agent_profile, + streaming=streaming, + message_count=message_count, + role_system_count=role_system_count, + role_user_count=role_user_count, + role_assistant_count=role_assistant_count, + role_tool_count=role_tool_count, + tools_kept=tools_kept, + tools_stripped=tools_stripped, + tool_names_requested=tool_names_requested, + max_tokens=max_tokens, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + finish_reason=finish_reason, + upstream_status=upstream_status, + step_index=step_index, + context_window_size_bytes=context_window_size_bytes, + active_file=active_file, + first_message_hash=first_message_hash, + chat_new_session_detected=chat_new_session_detected, + experiment_tool_access_enabled=experiment_tool_access_enabled, + experiment_approval_policy=experiment_approval_policy, + first_system_message=first_system_message, + last_user_message=last_user_message, + response_text=response_text, + tool_name=tool_name, + tool_arguments_length=tool_arguments_length, + tool_result_length=tool_result_length, + tool_arguments=tool_arguments, + tool_result=tool_result, + extra_json=extra_json, + payload_json=payload_json, + ) + if ignore_duplicate_source and source is not None and source_event_id is not None: + values = { + column.name: getattr(event, column.name) + for column in db_schemas.AgentEvent.__table__.columns + } + inserted_event_id = db.execute( + pg_insert(db_schemas.AgentEvent) + .values(**values) + .on_conflict_do_nothing(constraint="uq_agent_event_source_identity") + .returning(db_schemas.AgentEvent.event_id) + ).scalar_one_or_none() + if inserted_event_id is None: + return None + if commit: + db.commit() + return db.get(db_schemas.AgentEvent, inserted_event_id) + db.add(event) + if commit: + db.commit() + db.refresh(event) + else: + db.flush() + return event + + +# ── Agent edits (human-in-the-loop decisions) ─────────────────────────────── + + +def get_agent_memory_by_session_id( + db: Session, session_id: str +) -> Optional[db_schemas.AgentMemory]: + return ( + db.query(db_schemas.AgentMemory) + .filter(db_schemas.AgentMemory.session_id == session_id) + .first() + ) + + +def upsert_agent_memory( + db: Session, + *, + session_id: str, + owner_user_id: str, + owner_project_id: str, + messages: List[dict], +) -> db_schemas.AgentMemory: + """Replace the stored memory snapshot for one agent session. + + Ownership is written from the server-derived ACP scope, never from the + request body, so a client cannot attach its memory to someone else's + session. + """ + memory_json = json.dumps({"messages": messages}, sort_keys=True) + existing_memory = get_agent_memory_by_session_id(db, session_id) + now = datetime.now() + if existing_memory is None: + agent_memory = db_schemas.AgentMemory( + session_id=session_id, + owner_user_id=owner_user_id, + owner_project_id=owner_project_id, + memory_json=memory_json, + created_at=now, + updated_at=now, + ) + db.add(agent_memory) + else: + existing_memory.owner_user_id = owner_user_id + existing_memory.owner_project_id = owner_project_id + existing_memory.memory_json = memory_json + existing_memory.updated_at = now + agent_memory = existing_memory + db.commit() + db.refresh(agent_memory) + return agent_memory + + +def delete_agent_memory(db: Session, session_id: str) -> bool: + result = ( + db.query(db_schemas.AgentMemory) + .filter(db_schemas.AgentMemory.session_id == session_id) + .delete() + ) + db.commit() + return result > 0 diff --git a/src/database/db_schemas.py b/src/database/db_schemas.py index 7d0ee69..49cd286 100644 --- a/src/database/db_schemas.py +++ b/src/database/db_schemas.py @@ -13,7 +13,10 @@ - Session management and tracking """ -from datetime import datetime +import json +from datetime import datetime, timezone +from decimal import Decimal +from enum import Enum from pgvector.sqlalchemy import Vector from sqlalchemy import ( @@ -27,11 +30,15 @@ ForeignKey, Index, Integer, + Numeric, PrimaryKeyConstraint, String, Text, + UniqueConstraint, + func, + text, ) -from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import relationship from .db import Base @@ -42,8 +49,33 @@ "store_context": False, # Whether to store code context "store_contextual_telemetry": True, # Whether to collect contextual data "store_behavioral_telemetry": True, # Whether to collect behavioral data + # Whether to store agent task/event/edit *content* (message text, tool + # arguments/results, diffs). Structural telemetry (tokens, latency, span + # tree, tool names) is always stored and is not covered by this flag. + # + # Default ON: this is a research platform where study participants give + # informed consent through a separate process before using it, so opt-out + # is the right default for research data collection. The *mechanism* stays + # strict — enforcement is server-side only (see + # backend.routers.agent.consent.resolve_store_agent_content); a + # client-supplied flag is never trusted. + "store_agent_content": True, } +# Preference key gating persistence of agent content columns. +STORE_AGENT_CONTENT_KEY = "store_agent_content" +# Server-side default applied when a user row predates this preference (i.e. the +# key is absent from their stored preference JSON). +STORE_AGENT_CONTENT_DEFAULT = True + + +class ResearchStudyStatus(str, Enum): + """Lifecycle states for research studies on the shared ``study`` table.""" + + DRAFT = "DRAFT" + ACTIVE = "ACTIVE" + STUDY_STOPPED = "STUDY_STOPPED" + class Config(Base): """ @@ -79,6 +111,8 @@ class User(Base): Index("idx_user_config_id", "config_id"), # Index on admin status for efficient filtering Index("idx_user_is_admin", "is_admin"), + # Index on researcher enablement for admin/ownership lookups. + Index("idx_user_can_research", "can_research"), {"schema": "public"}, ) @@ -105,6 +139,18 @@ class User(Base): is_admin = Column( Boolean, server_default="false", default=False ) # Admin status for access control + # Researcher enablement (P3): an administrator flips this on to let an + # account own private profiles and studies. It is the *only* researcher + # authority — there is no per-study role table. Never settable by the + # account itself (no self-promotion route). + can_research = Column( + Boolean, server_default="false", default=False, nullable=False + ) + # When the account opted out of data collection (GDPR consent withdrawal); + # NULL while collection follows the preferences. Owned by the privacy + # endpoints only: the generic preference update cannot touch it, so a stale + # client pushing its local preferences never re-enables collection. + data_collection_opted_out_at = Column(DateTime(timezone=True), nullable=True) # Relationship to configuration data config = relationship("Config") @@ -452,6 +498,24 @@ class MetaQuery(Base): Index("idx_meta_query_project_id", "project_id"), Index("idx_meta_query_session_id", "session_id"), Index("idx_meta_query_type", "query_type"), + Index("idx_meta_query_timestamp", "timestamp"), + Index("idx_meta_query_timestamp_type", "timestamp", "query_type"), + # Foreign keys to context/telemetry: their deletes (erasure) check here. + Index( + "idx_meta_query_context_id", + "context_id", + postgresql_where=text("context_id IS NOT NULL"), + ), + Index( + "idx_meta_query_contextual_telemetry_id", + "contextual_telemetry_id", + postgresql_where=text("contextual_telemetry_id IS NOT NULL"), + ), + Index( + "idx_meta_query_behavioral_telemetry_id", + "behavioral_telemetry_id", + postgresql_where=text("behavioral_telemetry_id IS NOT NULL"), + ), {"schema": "public"}, ) @@ -557,6 +621,9 @@ class HadGeneration(Base): PrimaryKeyConstraint("meta_query_id", "model_id"), # Index for efficient query-model lookups Index("idx_had_generation_meta_query_model", "meta_query_id", "model_id"), + Index("idx_had_generation_confidence", "confidence"), + Index("idx_had_generation_model_id", "model_id"), + Index("idx_had_generation_was_accepted", "was_accepted"), {"schema": "public"}, ) @@ -577,7 +644,7 @@ class HadGeneration(Base): was_accepted = Column( Boolean, nullable=False ) # Whether user accepted the completion - confidence = Column(Double, nullable=False) # Model confidence score + confidence = Column(Double, nullable=True) # Model confidence score; NULL when the model reports none logprobs = Column(ARRAY(Double), nullable=False) # Log probabilities for tokens @@ -631,36 +698,86 @@ class Documentation(Base): embedding = Column( Vector(384), nullable=True ) # 384 dimensions for all-MiniLM-L6-v2 - created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + created_at = Column( + DateTime(timezone=True), nullable=False, default=datetime.now, server_default=func.now() + ) class Study(Base): """ - A/B testing and user experiment management. - - Manages user studies for configuration testing and analysis. - Only one study can be active at a time per server instance. + Shared study identity for research studies (plus pre-existing legacy rows). + + Research studies set ``is_research`` True and are owner-scoped; their + lifecycle and fixed configuration live in the research-only columns below. + ``default_config_id`` is nullable and not fabricated for a research study + (validated in code, not by a NOT NULL column). ``is_active`` mirrors the + ``research_status`` lifecycle projection (``ACTIVE`` ↔ True) and is not an + owner-level publication slot, so an owner may hold several active research + studies at once. Pre-existing legacy completion rows leave those research + columns nullable and continue using the existing completion fields; no new + completion A/B studies are created. """ - + __tablename__ = "study" __table_args__ = ( Index("idx_study_is_active", "is_active"), Index("idx_study_created_by", "created_by"), Index("idx_study_starts_at", "starts_at"), Index("idx_study_ends_at", "ends_at"), + Index("idx_study_research_status", "research_status"), + Index( + "uq_study_research_join_code", + "join_code", + unique=True, + postgresql_where=text("is_research AND join_code IS NOT NULL"), + ), {"schema": "public"}, ) - + study_id = Column(UUID(as_uuid=True), primary_key=True) name = Column(Text, nullable=False) description = Column(Text) - created_by = Column(UUID(as_uuid=True), ForeignKey("public.user.user_id"), nullable=False) + created_by = Column( + UUID(as_uuid=True), + ForeignKey("public.user.user_id", ondelete="CASCADE"), + nullable=False, + ) starts_at = Column(DateTime(timezone=True), nullable=False) ends_at = Column(DateTime(timezone=True), nullable=True) is_active = Column(Boolean, server_default="false", default=False) - default_config_id = Column(BigInteger, ForeignKey("public.config.config_id"), nullable=False) - created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) - + # Nullable at the DB level: required for completion studies, absent for + # agent research studies (enforced by request validation). + default_config_id = Column( + BigInteger, ForeignKey("public.config.config_id"), nullable=True + ) + # True for researcher-authored studies; False for legacy completion studies. + is_research = Column(Boolean, server_default="false", default=False, nullable=False) + # Research-only lifecycle/configuration authority. Legacy completion rows + # leave these nullable and continue using their existing fields. + research_status = Column(String, nullable=True) + research_config_json = Column(JSONB, nullable=True) + research_config_digest = Column(String, nullable=True) + join_code = Column(String, nullable=True) + consent_locked_at = Column(DateTime(timezone=True), nullable=True) + stopped_at = Column(DateTime(timezone=True), nullable=True) + stopped_by = Column(String, nullable=True) + # Participant inference budgets (shared provider key). Editable at any time + # and deliberately outside the digested ``research_config_json``. The default + # is 0 so an unconfigured study cannot spend (fail closed); balances for new + # enrollments copy this default. ``warning_fraction`` drives the participant + # warning surfaces only. + inference_budget_default_micro_usd = Column( + BigInteger, nullable=False, server_default="0", default=0 + ) + inference_budget_warning_fraction = Column( + Numeric(4, 3), nullable=False, server_default="0.800", default=Decimal("0.800") + ) + inference_budget_updated_at = Column(DateTime(timezone=True), nullable=True) + inference_budget_updated_by = Column(String, nullable=True) + created_at = Column( + DateTime(timezone=True), nullable=False, default=datetime.now, server_default=func.now() + ) + # Relationships creator = relationship("User") default_config = relationship("Config") @@ -682,10 +799,20 @@ class ConfigAssignmentHistory(Base): {"schema": "public"}, ) - user_id = Column(UUID(as_uuid=True), ForeignKey("public.user.user_id"), primary_key=True) - study_id = Column(UUID(as_uuid=True), ForeignKey("public.study.study_id"), primary_key=True) + user_id = Column( + UUID(as_uuid=True), + ForeignKey("public.user.user_id", ondelete="CASCADE"), + primary_key=True, + ) + study_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study.study_id", ondelete="CASCADE"), + primary_key=True, + ) assigned_config_id = Column(BigInteger, ForeignKey("public.config.config_id"), nullable=False) - assigned_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + assigned_at = Column( + DateTime(timezone=True), nullable=False, default=datetime.now, server_default=func.now() + ) # Relationships user = relationship("User") @@ -693,6 +820,484 @@ class ConfigAssignmentHistory(Base): assigned_config = relationship("Config") +# ── Agent tables ────────────────────────────────────────────────────────────── +# +# One schema serves both agent runtimes: +# +# * the built-in ``code4me2-agent`` ReAct loop, which runs locally and +# self-reports its steps to POST /api/agent/events/ingest, and +# * third-party ACP agents (Goose, Codex), which are observed transparently +# by the plugin's local proxy relaying through POST /api/agent/inference. +# +# Shape: "columnar core + JSON overflow". Fields both runtimes produce get +# first-class typed columns (span tree, token counts, latency, tool names, +# finish reason); runtime-specific or experimental extras go into the single +# ``extra_json`` overflow column so adding an adapter never needs a migration. +# +# Privacy split: structural columns (status, timings, token counts, decisions) +# are always stored. Content columns (task_description, first_system_message, +# last_user_message, response_text, tool_arguments, tool_result, payload_json, +# diff_text, edit_delta_json) are nullable and only ever written when the +# requesting user's ``store_agent_content`` preference resolves True — checked +# server-side, never taken from a client-supplied flag. + + +class ProviderConnection(Base): + """An administrator-maintained upstream provider endpoint. + + A connection owns the provider endpoint (``base_url``) and the *name* of the + deployment secret that holds its key (``secret_ref`` — an environment + variable name, never the value). ``models_json`` is the JSON array of model + names the connection is allowed to serve. Researchers never see + ``base_url``/``secret_ref``; they select a granted connection by id and a + model from ``models_json``. The secret value is resolved from the + environment only at inference time and never persisted or returned. + """ + + __tablename__ = "provider_connection" + __table_args__ = ( + Index("idx_provider_connection_is_active", "is_active"), + {"schema": "public"}, + ) + + connection_id = Column(UUID(as_uuid=True), primary_key=True) + label = Column(String, nullable=False, unique=True) + base_url = Column(String, nullable=False) + # Name of the environment variable holding the upstream key — never the key. + secret_ref = Column(String, nullable=False) + # JSON array of allowed model names for this connection. + models_json = Column(Text, nullable=False) + is_active = Column(Boolean, server_default="true", default=True, nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + +class ProviderModelPrice(Base): + """Per-model prices for one provider connection (USD per million tokens). + + Prices size the worst-case hold and settle the actual charge of every + metered inference call. A metered model without a price row fails closed + (503 ``price_missing``); there is never a default price. ``usd_per_million`` + equals micro-USD per token, so ``cost_micro_usd = ceil(tokens * price)``. + ``cached_input_usd_per_million`` NULL means cached prompt tokens bill at + the input price. Prices are not secret: researchers see them to size + budgets; only administrators write them. + """ + + __tablename__ = "provider_model_price" + __table_args__ = ( + CheckConstraint("input_usd_per_million >= 0", name="ck_provider_model_price_input"), + CheckConstraint("output_usd_per_million >= 0", name="ck_provider_model_price_output"), + CheckConstraint( + "cached_input_usd_per_million IS NULL OR cached_input_usd_per_million >= 0", + name="ck_provider_model_price_cached_input", + ), + Index("idx_provider_model_price_connection", "connection_id"), + {"schema": "public"}, + ) + + connection_id = Column( + UUID(as_uuid=True), + ForeignKey("public.provider_connection.connection_id", ondelete="CASCADE"), + primary_key=True, + ) + model = Column(String, primary_key=True) + input_usd_per_million = Column(Numeric(14, 6), nullable=False) + output_usd_per_million = Column(Numeric(14, 6), nullable=False) + cached_input_usd_per_million = Column(Numeric(14, 6), nullable=True) + updated_at = Column( + DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) + ) + updated_by = Column(String, nullable=True) + + +class AgentProfile(Base): + """A researcher-owned, editable agent configuration template. + + A profile fixes the runtime, the provider connection/model, the selected + tools, the release artifact pin and the approval policy. It is a + researcher-owned template whose configuration digest is refreshed on edit; + study selections store their own immutable snapshot. ``connection_id`` names an administrator-managed + :class:`ProviderConnection`; the provider endpoint and secret live there, + never on the profile. + """ + + __tablename__ = "agent_profile" + __table_args__ = ( + Index("idx_agent_profile_is_active", "is_active"), + Index("idx_agent_profile_owner_user_id", "owner_user_id"), + # Profile names are unique per researcher, never globally. + UniqueConstraint("owner_user_id", "name", name="uq_agent_profile_owner_name"), + {"schema": "public"}, + ) + + profile_id = Column(UUID(as_uuid=True), primary_key=True) + # The researcher who owns this private profile template. + owner_user_id = Column( + UUID(as_uuid=True), + ForeignKey("public.user.user_id", ondelete="CASCADE"), + nullable=False, + ) + name = Column(String, nullable=False) + model = Column(String, nullable=False) + # Agent runtime this profile targets: code4me2-agent | goose | codex + framework_version = Column( + String, nullable=False, server_default="code4me2-agent" + ) + # The exact approved artifact this profile runs (agent_release.release_id). + # The release row owns artifact identity; the profile only pins it. + release_id = Column( + String, + ForeignKey("public.agent_release.release_id", ondelete="RESTRICT"), + nullable=True, + ) + # Administrator-managed provider endpoint + secret owner. + connection_id = Column( + UUID(as_uuid=True), + ForeignKey("public.provider_connection.connection_id", ondelete="RESTRICT"), + nullable=True, + ) + tools_json = Column(Text, nullable=False) # JSON array of tool names + approval_policy = Column(String, nullable=False) # suggestion_only | per_step | … + max_steps = Column(Integer, nullable=False) + # Sampling temperature injected into the upstream request, server-side, the same + # way `model` is overridden. NULL = don't inject; let the provider use its default. + temperature = Column(Double, nullable=True) + max_context_tokens = Column( + Integer, nullable=True + ) # per-turn rolling window; NULL = model max + # Researcher-authored system prompt (managed runtime only; BYOA refuses it). + # NULL = no prompt: the runtime keeps its built-in one. It joins the + # configuration digest and study snapshots only when set, so profiles + # without a prompt keep their existing digests. + system_prompt = Column(Text, nullable=True) + # Built-in runtime command and harness settings (decision D-01; managed + # runtime only, BYOA refuses them), validated by ``agents.tools``. NULL = + # not set: no command is blocked (a user's config row may still block + # some), the runtime's default command timeout and the runtime's harness + # defaults. Like ``system_prompt`` each joins the configuration digest and + # study snapshots only when set. Read the JSON columns through + # ``commands_denylist`` / ``harness_options``. The retired + # ``commands_allowlist_json`` column (revision 8a0084080b46) stays in the + # table, unmapped, so an older backend can still be rolled back to. + commands_denylist_json = Column(Text, nullable=True) # JSON array of bare command names + command_timeout_seconds = Column(Integer, nullable=True) # 1..600 seconds + harness_options_json = Column(Text, nullable=True) # JSON object of switches + configuration_digest = Column(String, nullable=False, server_default="") + # Only active profiles are candidates for new study selections. Inactive + # profiles stay in the table for historical snapshots. + is_active = Column(Boolean, server_default="true", default=True, nullable=False) + created_at = Column(DateTime, default=datetime.now) + + owner = relationship("User") + connection = relationship("ProviderConnection") + + @property + def commands_denylist(self): + """The decoded ``commands_denylist_json`` (``None`` when unset).""" + return _decode_optional_json_text(self.commands_denylist_json) + + @property + def harness_options(self): + """The decoded ``harness_options_json`` (``None`` when unset).""" + return _decode_optional_json_text(self.harness_options_json) + + +def _decode_optional_json_text(raw): + """Decode an optional JSON text column. + + Text that is not JSON (only possible when a row is written around the API) + is returned verbatim rather than read as "unset", so every validator + downstream refuses it instead of silently dropping an arm's setting. + """ + if raw is None: + return None + try: + return json.loads(raw) + except (TypeError, ValueError): + return raw + + +class AgentTask(Base): + """One row per agent session a developer starts. + + A "session" here is one run of whichever agent runtime the assigned profile + selects — Goose, Codex, or the built-in ``code4me2-agent`` ReAct loop. + + The profile's config (model, temperature, approval_policy, tools_json) is + *snapshotted* onto this row at creation time, so later edits to the profile + never retroactively change the profile configuration a completed task ran under. + """ + + __tablename__ = "agent_task" + __table_args__ = ( + Index("idx_agent_task_status", "status"), + Index("idx_agent_task_session_id", "session_id"), + Index("idx_agent_task_owner_user_id", "owner_user_id"), + Index("idx_agent_task_study_id", "study_id"), + Index("idx_agent_task_study_assignment_id", "study_assignment_id"), + Index("idx_agent_task_profile_id", "profile_id"), + Index("idx_agent_task_research_session_id", "research_session_id"), + Index("idx_agent_task_enrollment_id", "enrollment_id"), + {"schema": "public"}, + ) + + task_id = Column(UUID(as_uuid=True), primary_key=True) + # nullable — populated when the task is minted via the authenticated plugin endpoint + session_id = Column( + UUID(as_uuid=True), ForeignKey("public.session.session_id"), nullable=True + ) + # Which path created this task: "plugin" (proxy relay, third-party agents), + # "code4me2_agent" (self-reporting built-in runtime), or "benchmark". + source = Column(String, nullable=False, server_default="plugin") + # Owner attribution for tasks that arrive over the ACP bearer path, where + # there is no session cookie to resolve the user from. + owner_user_id = Column( + UUID(as_uuid=True), ForeignKey("public.user.user_id"), nullable=True + ) + # The researcher whose provider connection funds this task's inference. + # Distinct from owner_user_id, which is the participant who ran the task. + funding_owner_user_id = Column( + UUID(as_uuid=True), + ForeignKey("public.user.user_id", ondelete="SET NULL"), + nullable=True, + ) + owner_project_id = Column(UUID(as_uuid=True), nullable=True) + # The runtime's *own* identifiers, which are not guaranteed to be UUIDs + # (code4me2-agent uses uuid4().hex; ACP session ids are opaque strings). + # Kept alongside the server-side UUID PK so self-reported batches can be + # matched back to their task without forcing the runtime to adopt our ids. + external_run_id = Column(String, unique=True, nullable=True) + agent_session_id = Column(String, nullable=True) + study_id = Column(UUID(as_uuid=True), ForeignKey("public.study.study_id"), nullable=True) + study_assignment_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study_assignment.assignment_id", ondelete="SET NULL"), + nullable=True, + ) + # Explicit phase-05 research attribution. Resolved server-side from the + # authorized account + frozen assignment; NULL for ordinary non-study use + # (an explicit "no research context" state, never a guessed one). + research_session_id = Column(UUID(as_uuid=True), nullable=True) + enrollment_id = Column(UUID(as_uuid=True), nullable=True) + profile_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_profile.profile_id"), + nullable=True, + ) + agent_profile = Column(String, nullable=False) + model = Column(String, nullable=False) + # Snapshotted from the assigned profile at task-creation time, like `model`, so + # the value is stable for the life of the task. NULL = provider default. + temperature = Column(Double, nullable=True) + approval_policy = Column(String, nullable=False) + tools_json = Column(Text, nullable=False) + # Agent runtime (e.g. "goose", "codex", "code4me2-agent") — snapshotted from + # the profile, but left nullable so the proxy can backfill the concrete + # version string it observes ("goose 1.x") on the first inference call. + framework_version = Column(String, nullable=True) + # Complete, versioned managed-runtime policy. Nullable for legacy tasks. + policy_snapshot = Column(JSONB, nullable=True) + observed_framework_version = Column(String, nullable=True) + observed_tools_json = Column(Text, nullable=True) + consent_content_storage = Column(Boolean, nullable=True) + next_event_index = Column(Integer, nullable=False, server_default="0") + status = Column(String, nullable=False) # pending | running | done | failed + created_at = Column(DateTime, default=datetime.now) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime, nullable=True) + total_steps = Column(Integer, nullable=True) + input_tokens = Column(Integer, nullable=True) + output_tokens = Column(Integer, nullable=True) + # Points at the most recent event, so a dashboard can show "where is this + # task now" without scanning agent_event. + # + # agent_task and agent_event reference each other, which is a deliberate + # cycle. `use_alter=True` tells SQLAlchemy to emit this constraint as a + # separate ALTER after both tables exist, so metadata.create_all() can + # order the DDL instead of erroring on the cycle. The Alembic migration + # does the same thing by hand. + latest_event_id = Column( + UUID(as_uuid=True), + ForeignKey( + "public.agent_event.event_id", + use_alter=True, + name="agent_task_latest_event_id_fkey", + ), + nullable=True, + ) + # Content — written only when store_agent_content resolves True + task_description = Column(Text, nullable=True) + + study = relationship("Study") + profile = relationship("AgentProfile") + + +class AgentEvent(Base): + """One row per agent step, from either telemetry path. + + ``event_type`` is the unified vocabulary both paths emit into: + ``model_call`` and ``tool_call`` are the two that carry typed columns; + ``thought`` / ``observation`` / ``run_started`` / ``run_completed`` / + ``decision`` are structural markers. + + Rows form a span tree via ``span_id`` / ``parent_span_id`` (tool calls nest + under the model_call whose response requested them). Anything a specific + adapter reports that has no typed column lands in ``extra_json``. + """ + + __tablename__ = "agent_event" + __table_args__ = ( + Index("idx_agent_event_task_id", "task_id"), + Index("idx_agent_event_event_type", "event_type"), + Index("idx_agent_event_created_at", "created_at"), + UniqueConstraint("task_id", "event_index", name="uq_agent_event_task_event_index"), + UniqueConstraint( + "task_id", "source", "source_event_id", name="uq_agent_event_source_identity" + ), + {"schema": "public"}, + ) + + event_id = Column(UUID(as_uuid=True), primary_key=True) + task_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_task.task_id", ondelete="CASCADE"), + nullable=False, + ) + event_index = Column(Integer, nullable=False) + # model_call | tool_call | thought | observation | test_run | decision | + # run_started | run_completed | accepted | rejected + event_type = Column(String, nullable=False) + # Which telemetry path wrote this row: "proxy" (client-side relay observing + # a third-party agent) or "code4me2_agent" (runtime self-report). + source = Column(String, nullable=True) + source_event_id = Column(String, nullable=True) + # Envelope schema version reported by a self-reporting runtime, so old + # batches stay interpretable after the envelope evolves. + schema_version = Column(String, nullable=True) + latency_ms = Column(Integer, nullable=True) + created_at = Column(DateTime, default=datetime.now) + # Client-side event timestamp, as reported by a self-reporting runtime. + # Distinct from created_at, which is when the server persisted the row. + occurred_at = Column(DateTime(timezone=True), nullable=True) + + # Shared span identifiers (model_call and tool_call). + # Events are grouped by task via task_id; there is no separate trace_id column — + # it was always a duplicate of task_id. + span_id = Column(String, nullable=True) + parent_span_id = Column(UUID(as_uuid=True), nullable=True) + # Correlates every event emitted while serving one user prompt. + request_id = Column(String, nullable=True) + chat_session_index = Column(Integer, nullable=True) + + # ── model_call fields — null for tool_call rows ── + model = Column(String, nullable=True) + agent_profile = Column(String, nullable=True) + streaming = Column(Boolean, nullable=True) + message_count = Column(Integer, nullable=True) + # Per-role message counts (the shape of the context window). + role_system_count = Column(Integer, nullable=True) + role_user_count = Column(Integer, nullable=True) + role_assistant_count = Column(Integer, nullable=True) + role_tool_count = Column(Integer, nullable=True) + tools_kept = Column(Integer, nullable=True) + tools_stripped = Column(Integer, nullable=True) + tool_names_requested = Column(ARRAY(String), nullable=True) + max_tokens = Column(Integer, nullable=True) + prompt_tokens = Column(Integer, nullable=True) + completion_tokens = Column(Integer, nullable=True) + total_tokens = Column(Integer, nullable=True) + finish_reason = Column(String, nullable=True) + upstream_status = Column(Integer, nullable=True) + step_index = Column(Integer, nullable=True) + context_window_size_bytes = Column(Integer, nullable=True) + active_file = Column(Text, nullable=True) + first_message_hash = Column(String, nullable=True) + chat_new_session_detected = Column(Boolean, nullable=True) + experiment_tool_access_enabled = Column(Boolean, nullable=True) + experiment_approval_policy = Column(String, nullable=True) + # Content — written only when store_agent_content resolves True + first_system_message = Column(Text, nullable=True) + last_user_message = Column(Text, nullable=True) + response_text = Column(Text, nullable=True) + + # ── tool_call fields — null for model_call rows ── + tool_name = Column(String, nullable=True) + tool_arguments_length = Column(Integer, nullable=True) + tool_result_length = Column(Integer, nullable=True) + # Content — written only when store_agent_content resolves True + tool_arguments = Column(Text, nullable=True) + tool_result = Column(Text, nullable=True) + + # ── JSON overflow ── + # Adapter-specific / experimental structural metrics that don't warrant a + # typed column (e.g. the ReAct adapter's per-iteration counters, an + # adapter's own observability block). Never content — see payload_json. + extra_json = Column(Text, nullable=True) + # Free-form event payload from a self-reporting runtime. May quote user + # prompts, model output, or file contents, so this is Content — written only + # when store_agent_content resolves True. + payload_json = Column(Text, nullable=True) + + +class AgentEdit(Base): + """One row per file the agent proposed changing. + + The accept / reject / modified decision plus the diff is the + human-in-the-loop signal: it records not just what the agent suggested but + what the developer actually did with it. + """ + + __tablename__ = "agent_edit" + __table_args__ = ( + Index("idx_agent_edit_task_id", "task_id"), + {"schema": "public"}, + ) + + edit_id = Column(UUID(as_uuid=True), primary_key=True) + task_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_task.task_id", ondelete="CASCADE"), + nullable=False, + ) + file_path = Column(Text, nullable=False) + was_accepted = Column(Boolean, nullable=True) # True/False after developer decides + was_modified = Column( + Boolean, nullable=True + ) # True if developer edited before accepting + decided_at = Column(DateTime, nullable=True) + # Content — written only when store_agent_content resolves True + diff_text = Column(Text, nullable=True) + edit_delta_json = Column( + Text, nullable=True + ) # what developer changed before accepting + + +class AgentMemory(Base): + """Durable per-session agent memory snapshot. + + Holds the latest serialized memory window for one agent chat session, so + conversation state survives an IDE (or agent process) restart. Ownership is + server-derived from the ACP token, matching agent telemetry authorization — + the client cannot claim a session it doesn't own. + """ + + __tablename__ = "agent_memory" + __table_args__ = ( + Index("idx_agent_memory_owner_user_id", "owner_user_id"), + Index("idx_agent_memory_owner_project_id", "owner_project_id"), + Index("idx_agent_memory_updated_at", "updated_at"), + {"schema": "public"}, + ) + + # ACP session ids are opaque strings, not guaranteed to be UUIDs. + session_id = Column(String, primary_key=True) + owner_user_id = Column(String, nullable=False) + owner_project_id = Column(String, nullable=False) + memory_json = Column(Text, nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + updated_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + # # class SessionQuery(Base): # __tablename__ = "session_queries" @@ -720,3 +1325,11 @@ class ConfigAssignmentHistory(Base): # UniqueConstraint("session_id", "query_id", name="unique_session_query"), # Index("idx_session_queries_query_id", "query_id"), # ) + + +# ``AgentProfile.release_id`` references the research ``agent_release`` table by +# string name. Importing the research namespace here registers its tables on the +# shared ``Base`` metadata, so ordering/creating the operational metadata works +# regardless of which namespace an entrypoint imports first. ``research_schemas`` +# never imports ``db_schemas``, so this is not an import cycle. +from . import research_schemas as _research_schemas # noqa: E402,F401 (registers research tables on Base) diff --git a/src/database/init.sql b/src/database/init.sql index e26ddbd..fc27daf 100644 --- a/src/database/init.sql +++ b/src/database/init.sql @@ -174,7 +174,7 @@ CREATE TABLE IF NOT EXISTS public.had_generation generation_time integer NOT NULL, shown_at timestamp with time zone[] NOT NULL, was_accepted boolean NOT NULL, - confidence double precision NOT NULL, + confidence double precision, logprobs double precision[] NOT NULL, PRIMARY KEY (meta_query_id, model_id) ); @@ -423,6 +423,11 @@ CREATE INDEX IF NOT EXISTS idx_meta_query_session_id ON public.meta_query (sessi CREATE INDEX IF NOT EXISTS idx_meta_query_type ON public.meta_query (query_type); CREATE INDEX IF NOT EXISTS idx_meta_query_timestamp ON public.meta_query ("timestamp"); CREATE INDEX IF NOT EXISTS idx_meta_query_timestamp_type ON public.meta_query ("timestamp", query_type); +-- The context and telemetry foreign keys: deleting a context/telemetry row +-- (account erasure) checks meta_query for references, which scans without these. +CREATE INDEX IF NOT EXISTS idx_meta_query_context_id ON public.meta_query (context_id) WHERE context_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_meta_query_contextual_telemetry_id ON public.meta_query (contextual_telemetry_id) WHERE contextual_telemetry_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_meta_query_behavioral_telemetry_id ON public.meta_query (behavioral_telemetry_id) WHERE behavioral_telemetry_id IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_chat_query_chat_id ON public.chat_query (chat_id); @@ -678,12 +683,6 @@ INSERT INTO public.config (config_data) VALUES ('config { contextPath = "" timeout = 5000 } - // Authentication Settings - auth { - google { - clientId = "288822392430-se6cmstnkje31e5kv32u95lo8544ltu4.apps.googleusercontent.com" - } - } // model configuration models { available = [ @@ -898,10 +897,12 @@ INSERT INTO public.config (config_data) VALUES ('config { } }'); +-- The plugin's default rows, 1 (completion) and 3 (chat), are served by OpenRouter +-- (key from $OPENROUTER_API_KEY). The other rows run locally and only when chosen. INSERT INTO public.model_name (model_name, is_instruction_tuned, prompt_templates, model_parameters) VALUES - ('deepseek-ai/deepseek-coder-1.3b-base', FALSE,'{"fim_template":{"multi_file_template":"{multi_file_context}#{file_name}\n<｜fim▁begin｜>{prefix}<｜fim▁hole｜>{suffix}<｜fim▁end｜>","single_file_template":"<｜fim▁begin｜>{prefix}<｜fim▁hole｜>{suffix}<｜fim▁end｜>"},"file_separator":"#{file_name}\n","stop_tokens":["\n\n"]}', '{"max_new_tokens": 64}'), + ('deepseek-ai/deepseek-coder-1.3b-base', FALSE,'{"fim_template":{"multi_file_template":"{multi_file_context}#{file_name}\n<｜fim▁begin｜>{prefix}<｜fim▁hole｜>{suffix}<｜fim▁end｜>","single_file_template":"<｜fim▁begin｜>{prefix}<｜fim▁hole｜>{suffix}<｜fim▁end｜>"},"file_separator":"#{file_name}\n","stop_tokens":["\n\n"]}', '{"provider": "openai_compatible", "kind": "completion", "base_url": "https://openrouter.ai/api/v1", "api_key_ref": "OPENROUTER_API_KEY", "provider_model": "mistralai/codestral-2508", "max_new_tokens": 64}'), ('bigcode/starcoder2-3b', FALSE,'{"fim_template":{"multi_file_template":"{multi_file_context}{file_name}\n{prefix}{suffix}","single_file_template":"{prefix}{suffix}"},"file_separator":"\n{file_name}\n","stop_tokens":["","\n\n"]}', '{"max_new_tokens": 64}'), - ('mistralai/Ministral-8B-Instruct-2410', TRUE, '{}', '{"max_new_tokens": 256}'), + ('mistralai/Ministral-8B-Instruct-2410', TRUE, '{}', '{"provider": "openai_compatible", "kind": "chat", "base_url": "https://openrouter.ai/api/v1", "api_key_ref": "OPENROUTER_API_KEY", "provider_model": "mistralai/ministral-8b-2512", "max_new_tokens": 256}'), ('JetBrains/Mellum-4b-base', FALSE, '{"fim_template":{"multi_file_template":"{multi_file_context}{file_name}\n{suffix}{prefix}","single_file_template":"{suffix}{prefix}"},"file_separator":"{file_name}\n","stop_tokens":["", "\n\n"]}', '{"max_new_tokens": 64}'); INSERT INTO public.programming_language (language_name) VALUES diff --git a/src/database/init_test.sql b/src/database/init_test.sql index 4c93cc7..9208821 100644 --- a/src/database/init_test.sql +++ b/src/database/init_test.sql @@ -22,7 +22,8 @@ CREATE TABLE IF NOT EXISTS public."user" verified BOOLEAN DEFAULT FALSE, config_id BIGINT NOT NULL, preference TEXT, - auth_token uuid NULL + auth_token uuid NULL, + is_admin BOOLEAN DEFAULT FALSE ); -- Model name table with instruction tuning flag @@ -173,7 +174,7 @@ CREATE TABLE IF NOT EXISTS public.had_generation generation_time integer NOT NULL, shown_at timestamp with time zone[] NOT NULL, was_accepted boolean NOT NULL, - confidence double precision NOT NULL, + confidence double precision, logprobs double precision[] NOT NULL, PRIMARY KEY (meta_query_id, model_id) ); @@ -198,6 +199,30 @@ CREATE TABLE IF NOT EXISTS public.documentation created_at timestamp with time zone NOT NULL DEFAULT NOW() ); +-- Study table for A/B testing and user experiments +CREATE TABLE IF NOT EXISTS public.study +( + study_id uuid NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + description TEXT, + created_by uuid NOT NULL, + starts_at timestamp with time zone NOT NULL, + ends_at timestamp with time zone, + is_active BOOLEAN DEFAULT FALSE, + default_config_id BIGINT NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT NOW() +); + +-- Config assignment history for tracking user treatments +CREATE TABLE IF NOT EXISTS public.config_assignment_history +( + user_id uuid NOT NULL, + study_id uuid NOT NULL, + assigned_config_id BIGINT NOT NULL, + assigned_at timestamp with time zone NOT NULL DEFAULT NOW(), + PRIMARY KEY (user_id, study_id) +); + -- Indexes for documentation table CREATE INDEX IF NOT EXISTS idx_documentation_language ON public.documentation (language); CREATE INDEX IF NOT EXISTS idx_documentation_embedding ON public.documentation @@ -341,9 +366,41 @@ ALTER TABLE public.session_projects REFERENCES public.project(project_id) ON DELETE CASCADE; +-- Foreign keys for study management +ALTER TABLE public.study + ADD CONSTRAINT fk_study_created_by FOREIGN KEY (created_by) + REFERENCES public."user" (user_id) + ON UPDATE NO ACTION + ON DELETE CASCADE; + +ALTER TABLE public.study + ADD CONSTRAINT fk_study_default_config FOREIGN KEY (default_config_id) + REFERENCES public.config (config_id) + ON UPDATE NO ACTION + ON DELETE NO ACTION; + +ALTER TABLE public.config_assignment_history + ADD CONSTRAINT fk_config_assignment_user FOREIGN KEY (user_id) + REFERENCES public."user" (user_id) + ON UPDATE NO ACTION + ON DELETE CASCADE; + +ALTER TABLE public.config_assignment_history + ADD CONSTRAINT fk_config_assignment_study FOREIGN KEY (study_id) + REFERENCES public.study (study_id) + ON UPDATE NO ACTION + ON DELETE CASCADE; + +ALTER TABLE public.config_assignment_history + ADD CONSTRAINT fk_config_assignment_config FOREIGN KEY (assigned_config_id) + REFERENCES public.config (config_id) + ON UPDATE NO ACTION + ON DELETE NO ACTION; + -- Indexes for performance CREATE INDEX IF NOT EXISTS idx_user_email ON public."user" (email); CREATE INDEX IF NOT EXISTS idx_user_config_id ON public."user" (config_id); +CREATE INDEX IF NOT EXISTS idx_user_is_admin ON public."user" (is_admin); CREATE INDEX IF NOT EXISTS idx_ctxt_telemetry_version_id ON public.contextual_telemetry (version_id); CREATE INDEX IF NOT EXISTS idx_ctxt_telemetry_trigger_type_id ON public.contextual_telemetry (trigger_type_id); @@ -359,17 +416,38 @@ CREATE INDEX IF NOT EXISTS idx_session_projects_project_id ON public.session_pro CREATE INDEX IF NOT EXISTS idx_chat_project_id ON public.chat (project_id); CREATE INDEX IF NOT EXISTS idx_chat_user_id ON public.chat (user_id); +-- Analytics-optimized indexes CREATE INDEX IF NOT EXISTS idx_meta_query_user_id ON public.meta_query (user_id); CREATE INDEX IF NOT EXISTS idx_meta_query_project_id ON public.meta_query (project_id); CREATE INDEX IF NOT EXISTS idx_meta_query_session_id ON public.meta_query (session_id); CREATE INDEX IF NOT EXISTS idx_meta_query_type ON public.meta_query (query_type); +CREATE INDEX IF NOT EXISTS idx_meta_query_timestamp ON public.meta_query ("timestamp"); +CREATE INDEX IF NOT EXISTS idx_meta_query_timestamp_type ON public.meta_query ("timestamp", query_type); +-- The context and telemetry foreign keys: deleting a context/telemetry row +-- (account erasure) checks meta_query for references, which scans without these. +CREATE INDEX IF NOT EXISTS idx_meta_query_context_id ON public.meta_query (context_id) WHERE context_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_meta_query_contextual_telemetry_id ON public.meta_query (contextual_telemetry_id) WHERE contextual_telemetry_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_meta_query_behavioral_telemetry_id ON public.meta_query (behavioral_telemetry_id) WHERE behavioral_telemetry_id IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_chat_query_chat_id ON public.chat_query (chat_id); CREATE INDEX IF NOT EXISTS idx_had_generation_meta_query_model ON public.had_generation (meta_query_id, model_id); +CREATE INDEX IF NOT EXISTS idx_had_generation_model_id ON public.had_generation (model_id); +CREATE INDEX IF NOT EXISTS idx_had_generation_was_accepted ON public.had_generation (was_accepted); +CREATE INDEX IF NOT EXISTS idx_had_generation_confidence ON public.had_generation (confidence); CREATE INDEX IF NOT EXISTS idx_ground_truth_completion_query_timestamp ON public.ground_truth (completion_query_id, truth_timestamp); +-- Study management indexes +CREATE INDEX IF NOT EXISTS idx_study_is_active ON public.study (is_active); +CREATE INDEX IF NOT EXISTS idx_study_created_by ON public.study (created_by); +CREATE INDEX IF NOT EXISTS idx_study_starts_at ON public.study (starts_at); +CREATE INDEX IF NOT EXISTS idx_study_ends_at ON public.study (ends_at); + +CREATE INDEX IF NOT EXISTS idx_config_assignment_user_id ON public.config_assignment_history (user_id); +CREATE INDEX IF NOT EXISTS idx_config_assignment_study_id ON public.config_assignment_history (study_id); +CREATE INDEX IF NOT EXISTS idx_config_assignment_assigned_at ON public.config_assignment_history (assigned_at); + -- Insert default data INSERT INTO public.config (config_data) VALUES ('config { // module configuration @@ -605,12 +683,6 @@ INSERT INTO public.config (config_data) VALUES ('config { contextPath = "" timeout = 5000 } - // Authentication Settings - auth { - google { - clientId = "67736337656-un249ihklv5i93n033i1v46q12bfv5g2.apps.googleusercontent.com" - } - } // model configuration models { available = [ @@ -857,4 +929,4 @@ INSERT INTO public.plugin_version (version_name, ide_type, description) VALUES ('0.1.0j', 'JetBrains', 'enhanced version with chat support'), ('0.1.0v', 'VSCode', 'enhanced version with chat support'); -COMMIT; \ No newline at end of file +COMMIT; diff --git a/src/database/migration/env.py b/src/database/migration/env.py index 933d52a..064cc8c 100644 --- a/src/database/migration/env.py +++ b/src/database/migration/env.py @@ -19,8 +19,12 @@ src_dir = os.path.join(project_root, "src") sys.path.insert(0, src_dir) -# Import database models +# Import every model namespace that declares tables. ``db_schemas`` owns the +# operational (Code4Me) tables and ``research_schemas`` owns the research tables; +# both attach to the same ``Base`` from ``database.db``, so importing both is +# what makes autogenerate/``alembic check`` see the research tables at all. try: + from database import research_schemas as _research_schemas # noqa: F401 from database.db_schemas import Base target_metadata = Base.metadata @@ -67,6 +71,7 @@ def run_migrations_offline() -> None: dialect_opts={"paramstyle": "named"}, compare_type=True, compare_server_default=True, + include_schemas=True, ) with context.begin_transaction(): @@ -82,6 +87,7 @@ def run_migrations_online() -> None: configuration, prefix="sqlalchemy.", poolclass=pool.NullPool, + connect_args={"options": "-csearch_path=code4me_migration"}, ) with connectable.connect() as connection: @@ -90,6 +96,8 @@ def run_migrations_online() -> None: target_metadata=target_metadata, compare_type=True, compare_server_default=True, + include_schemas=True, + version_table_schema="public", ) with context.begin_transaction(): diff --git a/src/database/migration/migration_manager.py b/src/database/migration/migration_manager.py index 561c94f..3922c7d 100644 --- a/src/database/migration/migration_manager.py +++ b/src/database/migration/migration_manager.py @@ -20,13 +20,26 @@ from alembic import command from alembic.config import Config +from alembic.script import ScriptDirectory +from dotenv import load_dotenv from sqlalchemy import create_engine, text +from sqlalchemy.engine import make_url # Set up paths current_dir = Path(__file__).parent # src/database/migration project_root = current_dir.parent.parent.parent # project root src_dir = project_root / "src" sys.path.insert(0, str(src_dir)) +# Match src/main.py: operator-supplied environment variables win, while a +# repository/deployment .env fills values that Docker Compose does not export. +load_dotenv(project_root / ".env", override=False) + + +def _database_url_for_log(value: str) -> str: + try: + return make_url(value).render_as_string(hide_password=True) + except Exception: + return "" class MigrationManager: @@ -142,24 +155,21 @@ def initialize_from_sql(self) -> bool: return False def setup_migration_tracking(self) -> bool: - """Set up Alembic migration tracking.""" + """Set up Alembic tracking by stamping at the start of the existing chain.""" print("Setting up migration tracking...") try: - # Set database URL self.alembic_cfg.set_main_option("sqlalchemy.url", self.get_database_url()) - # Create EMPTY baseline migration (not autogenerated) - # This represents the current state from init.sql - init_file_name = "init_test.sql" if self.use_test_db else "init.sql" - command.revision( - self.alembic_cfg, - message=f"Baseline migration from {init_file_name}", - autogenerate=False, # Create empty migration - ) - - # Mark as current state - command.stamp(self.alembic_cfg, "head") + # Stamp at "base" (no revision) so the subsequent `upgrade head` runs + # every migration in order from the committed baseline + # (c2dc3e9cc1bc_baseline_migration_from_init_sql). + # + # Do NOT create a new revision file here: the baseline is already + # committed, so generating another one would put this database on a + # parallel dead-end branch and silently skip all real migrations — + # which is how the agent tables would go missing on a fresh install. + command.stamp(self.alembic_cfg, "base") print("Migration tracking set up") return True @@ -193,77 +203,90 @@ def ensure_initialized(self) -> bool: return True - def init_migrations(self) -> None: + def init_migrations(self) -> bool: """Initialize migrations with hybrid approach.""" db_type = "test database" if self.use_test_db else "main database" print(f"Initializing migration system for {db_type}...") - print(f"Database URL: {self.get_database_url()}") + print(f"Database URL: {_database_url_for_log(self.get_database_url())}") if self.ensure_initialized(): print("Migration system ready!") print("Use 'create' to add new migrations, 'migrate' to apply them") + return True else: print("Failed to initialize migration system") + return False - def create_migration(self, message: str) -> None: + def create_migration(self, message: str) -> bool: """Create a new migration.""" print(f"Creating migration: {message}") # Ensure system is initialized first if not self.ensure_initialized(): - return + return False try: self.alembic_cfg.set_main_option("sqlalchemy.url", self.get_database_url()) command.revision(self.alembic_cfg, message=message, autogenerate=True) print("Migration created") + return True except Exception as e: print(f"Error: {e}") + return False - def migrate(self) -> None: + def migrate(self) -> bool: """Apply all migrations.""" print("Applying migrations...") # Ensure system is initialized first if not self.ensure_initialized(): - return + return False try: self.alembic_cfg.set_main_option("sqlalchemy.url", self.get_database_url()) command.upgrade(self.alembic_cfg, "head") + if not self.is_at_expected_head(): + print("Migration command completed, but the database is not at the expected Alembic head") + return False print("Migrations applied") + return True except Exception as e: print(f"Error: {e}") + return False - def current(self) -> None: + def current(self) -> bool: """Show current revision.""" if not self.check_database_connection(): - return + return False try: self.alembic_cfg.set_main_option("sqlalchemy.url", self.get_database_url()) command.current(self.alembic_cfg) + return True except Exception as e: print(f"Error: {e}") + return False - def history(self) -> None: + def history(self) -> bool: """Show migration history.""" try: command.history(self.alembic_cfg) + return True except Exception as e: print(f"Error: {e}") + return False - def status(self) -> None: + def status(self) -> bool: """Show detailed status of database and migrations.""" db_type = "Test Database" if self.use_test_db else "Main Database" print(f"Migration System Status - {db_type}") print("=" * 50) - print(f"Database URL: {self.get_database_url()}") + print(f"Database URL: {_database_url_for_log(self.get_database_url())}") # Check database connection if not self.check_database_connection(): print("Database: Not connected") - return + return False print("Database: Connected") @@ -300,18 +323,40 @@ def status(self) -> None: ) version = result.scalar() print(f"Current version: {version}") + if not self.is_at_expected_head(): + print("Migration readiness: Not at the expected head") + return False + print("Migration readiness: At expected head") except Exception: print("Could not get current version") + return False else: print("Migration tracking: Not set up") + return False + return True - def reset(self) -> None: + def is_at_expected_head(self) -> bool: + """Require the deployed database revision set to equal the code's heads.""" + try: + expected = set(ScriptDirectory.from_config(self.alembic_cfg).get_heads()) + engine = create_engine(self.get_database_url()) + with engine.connect() as conn: + applied = { + str(row[0]) + for row in conn.execute(text("SELECT version_num FROM alembic_version")) + } + return bool(expected) and applied == expected + except Exception as error: + print(f"Could not verify migration head: {error}") + return False + + def reset(self) -> bool: """Reset database and reinitialize.""" db_type = "test database" if self.use_test_db else "main database" print(f"Resetting {db_type}...") if not self.check_database_connection(): - return + return False try: # Drop and recreate public schema @@ -324,13 +369,14 @@ def reset(self) -> None: print("Database reset") # Reinitialize - self.init_migrations() + return self.init_migrations() except Exception as e: print(f"Error during reset: {e}") + return False -def main(): +def main() -> int: """CLI interface.""" parser = argparse.ArgumentParser(description="Hybrid Migration Manager") parser.add_argument( @@ -357,25 +403,26 @@ def main(): if not args.command: parser.print_help() - return + return 0 manager = MigrationManager(use_test_db=args.test) if args.command == "init": - manager.init_migrations() + succeeded = manager.init_migrations() elif args.command == "status": - manager.status() + succeeded = manager.status() elif args.command == "create": - manager.create_migration(args.message) + succeeded = manager.create_migration(args.message) elif args.command == "migrate": - manager.migrate() + succeeded = manager.migrate() elif args.command == "current": - manager.current() + succeeded = manager.current() elif args.command == "history": - manager.history() + succeeded = manager.history() elif args.command == "reset": - manager.reset() + succeeded = manager.reset() + return 0 if succeeded else 1 if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/src/database/migration/versions/8a0084080b46_consolidated_schema.py b/src/database/migration/versions/8a0084080b46_consolidated_schema.py new file mode 100644 index 0000000..4531b3a --- /dev/null +++ b/src/database/migration/versions/8a0084080b46_consolidated_schema.py @@ -0,0 +1,722 @@ +"""consolidated schema + +Revision ID: 8a0084080b46 +Revises: +Create Date: 2026-09-18 07:43:13.099261 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '8a0084080b46' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Upgrade database schema. + + Fresh-schema only: this consolidated revision creates the research schema and + has no data-preserving upgrade path from the previous revision/condition + model. A database that already holds the previous research tables is refused + with an actionable message instead of failing halfway through the DDL. + """ + bind = op.get_bind() + inspector = sa.inspect(bind) + previous_research_tables = sorted( + table + for table in ( + "research_enrollment", + "research_session", + "research_event", + "study_assignment", + "study_agent_profile", + ) + if inspector.has_table(table, schema="public") + ) + if previous_research_tables: + raise RuntimeError( + "Fresh-schema release: this database already contains the previous research " + f"schema ({', '.join(previous_research_tables)}), so there is no in-place upgrade " + "path. Create a new database (or a new compose volume) and re-run the migration. " + "Preserving existing study data requires a separate, explicitly approved " + "migration task." + ) + + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('acp_capability_receipt', + sa.Column('receipt_id', sa.UUID(), nullable=False), + sa.Column('captured_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('ide_build', sa.String(), nullable=False), + sa.Column('ai_assistant_build', sa.String(), nullable=False), + sa.Column('plugin_version', sa.String(), nullable=False), + sa.Column('os', sa.String(), nullable=False), + sa.Column('arch', sa.String(), nullable=False), + sa.Column('agent_id', sa.String(), nullable=False), + sa.Column('agent_version', sa.String(), nullable=False), + sa.Column('adapter_version', sa.String(), nullable=True), + sa.Column('protocol_version', sa.String(), nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('content_hash', sa.String(), nullable=False), + sa.Column('receipt_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('receipt_id'), + sa.UniqueConstraint('content_hash'), + schema='public' + ) + op.create_index('idx_acp_capability_receipt_agent', 'acp_capability_receipt', ['agent_id', 'agent_version'], unique=False, schema='public') + op.create_index('idx_acp_capability_receipt_captured_at', 'acp_capability_receipt', ['captured_at'], unique=False, schema='public') + op.create_index('idx_acp_capability_receipt_status', 'acp_capability_receipt', ['status'], unique=False, schema='public') + op.create_table('agent_memory', + sa.Column('session_id', sa.String(), nullable=False), + sa.Column('owner_user_id', sa.String(), nullable=False), + sa.Column('owner_project_id', sa.String(), nullable=False), + sa.Column('memory_json', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('session_id'), + schema='public' + ) + op.create_index('idx_agent_memory_owner_project_id', 'agent_memory', ['owner_project_id'], unique=False, schema='public') + op.create_index('idx_agent_memory_owner_user_id', 'agent_memory', ['owner_user_id'], unique=False, schema='public') + op.create_index('idx_agent_memory_updated_at', 'agent_memory', ['updated_at'], unique=False, schema='public') + op.create_table('agent_release', + sa.Column('release_id', sa.String(), nullable=False), + sa.Column('agent_id', sa.String(), nullable=False), + sa.Column('source_manifest_digest', sa.String(), nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('release_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('release_id'), + sa.UniqueConstraint('agent_id', 'source_manifest_digest', name='uq_agent_release_agent_manifest'), + schema='public' + ) + op.create_index('idx_agent_release_agent_id', 'agent_release', ['agent_id'], unique=False, schema='public') + op.create_index('idx_agent_release_status', 'agent_release', ['status'], unique=False, schema='public') + op.create_table('provider_connection', + sa.Column('connection_id', sa.UUID(), nullable=False), + sa.Column('label', sa.String(), nullable=False), + sa.Column('base_url', sa.String(), nullable=False), + sa.Column('secret_ref', sa.String(), nullable=False), + sa.Column('models_json', sa.Text(), nullable=False), + sa.Column('is_active', sa.Boolean(), server_default='true', nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('connection_id'), + sa.UniqueConstraint('label'), + schema='public' + ) + op.create_index('idx_provider_connection_is_active', 'provider_connection', ['is_active'], unique=False, schema='public') + op.create_table('research_event', + sa.Column('event_id', sa.UUID(), nullable=False), + sa.Column('schema_version', sa.String(), nullable=False), + sa.Column('event_type', sa.String(), nullable=False), + sa.Column('source', sa.String(), nullable=False), + sa.Column('study_id', sa.UUID(), nullable=True), + sa.Column('enrollment_id', sa.UUID(), nullable=True), + sa.Column('research_session_id', sa.UUID(), nullable=True), + sa.Column('agent_run_id', sa.String(), nullable=True), + sa.Column('emitter_id', sa.String(), nullable=False), + sa.Column('emitter_sequence', sa.Integer(), nullable=False), + sa.Column('occurred_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('envelope_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('digest', sa.String(), nullable=False), + sa.Column('accepted_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('retention_state', sa.String(), server_default='RETAINED', nullable=False), + sa.Column('anonymized_at', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('event_id'), + sa.UniqueConstraint('research_session_id', 'emitter_id', 'emitter_sequence', name='uq_research_event_session_emitter_sequence'), + schema='public' + ) + op.create_index('idx_research_event_digest', 'research_event', ['digest'], unique=False, schema='public') + op.create_index('idx_research_event_emitter_id', 'research_event', ['emitter_id'], unique=False, schema='public') + op.create_index('idx_research_event_enrollment_id', 'research_event', ['enrollment_id'], unique=False, schema='public') + op.create_index('idx_research_event_event_type', 'research_event', ['event_type'], unique=False, schema='public') + op.create_index('idx_research_event_occurred_at', 'research_event', ['occurred_at'], unique=False, schema='public') + op.create_index('idx_research_event_session_id', 'research_event', ['research_session_id'], unique=False, schema='public') + op.create_index('idx_research_event_study_id', 'research_event', ['study_id'], unique=False, schema='public') + op.create_index('idx_research_event_agent_run_id', 'research_event', ['agent_run_id'], unique=False, schema='public') + op.create_index('idx_research_event_retention_state', 'research_event', ['retention_state'], unique=False, schema='public') + op.create_table('research_participant', + sa.Column('participant_id', sa.UUID(), nullable=False), + sa.Column('account_id', sa.UUID(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('participant_id'), + sa.UniqueConstraint('account_id'), + schema='public' + ) + op.create_index('idx_research_participant_account_id', 'research_participant', ['account_id'], unique=False, schema='public') + op.create_table('research_record', + sa.Column('record_id', sa.UUID(), nullable=False), + sa.Column('kind', sa.String(), nullable=False), + sa.Column('scope_type', sa.String(), nullable=True), + sa.Column('scope_id', sa.UUID(), nullable=True), + sa.Column('study_id', sa.UUID(), nullable=True), + sa.Column('actor', sa.String(), nullable=True), + sa.Column('occurred_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('payload_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.PrimaryKeyConstraint('record_id'), + schema='public' + ) + op.create_index('idx_research_record_kind', 'research_record', ['kind'], unique=False, schema='public') + op.create_index('idx_research_record_occurred_at', 'research_record', ['occurred_at'], unique=False, schema='public') + op.create_index('idx_research_record_scope', 'research_record', ['scope_type', 'scope_id'], unique=False, schema='public') + op.create_index('idx_research_record_study_id', 'research_record', ['study_id'], unique=False, schema='public') + op.create_table('research_retention_job', + sa.Column('job_id', sa.UUID(), nullable=False), + sa.Column('enrollment_id', sa.UUID(), nullable=False), + sa.Column('action', sa.String(), nullable=False), + sa.Column('state', sa.String(), nullable=False), + sa.Column('attempts', sa.Integer(), server_default='0', nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('started_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('last_error', sa.String(), nullable=True), + sa.Column('evidence_digest', sa.String(), nullable=True), + sa.Column('affected_events', sa.Integer(), server_default='0', nullable=False), + sa.Column('evidence_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.PrimaryKeyConstraint('job_id'), + sa.UniqueConstraint('enrollment_id', 'action', name='uq_research_retention_job_enrollment_action'), + schema='public' + ) + op.create_index('idx_research_retention_job_enrollment', 'research_retention_job', ['enrollment_id'], unique=False, schema='public') + op.create_index('idx_research_retention_job_state', 'research_retention_job', ['state'], unique=False, schema='public') + op.create_table('research_session', + sa.Column('session_id', sa.UUID(), nullable=False), + sa.Column('enrollment_id', sa.UUID(), nullable=False), + sa.Column('study_id', sa.UUID(), nullable=False), + sa.Column('context_id', sa.String(), nullable=False), + sa.Column('state', sa.String(), nullable=False), + sa.Column('opened_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('last_heartbeat_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('last_activity_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('closed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('close_reason', sa.String(), nullable=True), + sa.Column('resume_generation', sa.Integer(), server_default='0', nullable=False), + sa.Column('manifest_digest', sa.String(), nullable=False), + sa.Column('environment_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('transitions_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('session_id'), + schema='public' + ) + op.create_index('idx_research_session_context_id', 'research_session', ['context_id'], unique=False, schema='public') + op.create_index('idx_research_session_enrollment_id', 'research_session', ['enrollment_id'], unique=False, schema='public') + op.create_index('idx_research_session_state', 'research_session', ['state'], unique=False, schema='public') + op.create_index('idx_research_session_study_id', 'research_session', ['study_id'], unique=False, schema='public') + op.create_index('uq_research_session_active_context', 'research_session', ['enrollment_id', 'context_id'], unique=True, schema='public', postgresql_where=sa.text("state NOT IN ('ended', 'revoked')")) + op.create_table('research_agent_run', + sa.Column('agent_run_id', sa.UUID(), nullable=False), + sa.Column('research_session_id', sa.UUID(), nullable=True), + sa.Column('agent_release_id', sa.String(), nullable=True), + sa.Column('assignment_id', sa.UUID(), nullable=True), + sa.Column('agent_profile_id', sa.UUID(), nullable=True), + sa.Column('profile_digest', sa.String(), nullable=True), + sa.Column('profile_snapshot_json', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('started_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('ended_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('outcome', sa.String(), nullable=True), + sa.Column('snapshot_json', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('snapshot_captured_at', sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(['research_session_id'], ['public.research_session.session_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('agent_run_id'), + schema='public' + ) + op.create_index('idx_research_agent_run_session_id', 'research_agent_run', ['research_session_id'], unique=False, schema='public') + op.create_table('research_enrollment', + sa.Column('enrollment_id', sa.UUID(), nullable=False), + sa.Column('participant_id', sa.UUID(), nullable=False), + sa.Column('study_id', sa.UUID(), nullable=False), + sa.Column('participant_code', sa.String(), nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('revocation_epoch', sa.Integer(), server_default='0', nullable=False), + sa.Column('eligibility_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('enrolled_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('consent_accepted_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('retention_action', sa.String(), nullable=False), + sa.ForeignKeyConstraint(['participant_id'], ['public.research_participant.participant_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('enrollment_id'), + sa.UniqueConstraint('participant_id', 'study_id', name='uq_research_enrollment_participant_study'), + schema='public' + ) + op.create_index('idx_research_enrollment_participant_code', 'research_enrollment', ['participant_code'], unique=False, schema='public') + op.create_index('idx_research_enrollment_participant_id', 'research_enrollment', ['participant_id'], unique=False, schema='public') + op.create_index('idx_research_enrollment_status', 'research_enrollment', ['status'], unique=False, schema='public') + op.create_index('idx_research_enrollment_study_id', 'research_enrollment', ['study_id'], unique=False, schema='public') + op.create_index('uq_research_enrollment_active_participant', 'research_enrollment', ['participant_id'], unique=True, schema='public', postgresql_where=sa.text("status = 'ACTIVE'")) + op.create_table('agent_profile', + sa.Column('profile_id', sa.UUID(), nullable=False), + sa.Column('owner_user_id', sa.UUID(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('model', sa.String(), nullable=False), + sa.Column('framework_version', sa.String(), server_default='code4me2-agent', nullable=False), + sa.Column('release_id', sa.String(), nullable=True), + sa.Column('connection_id', sa.UUID(), nullable=True), + sa.Column('tools_json', sa.Text(), nullable=False), + sa.Column('approval_policy', sa.String(), nullable=False), + sa.Column('max_steps', sa.Integer(), nullable=False), + sa.Column('temperature', sa.Double(), nullable=True), + sa.Column('max_context_tokens', sa.Integer(), nullable=True), + sa.Column('system_prompt', sa.Text(), nullable=True), + sa.Column('commands_allowlist_json', sa.Text(), nullable=True), + sa.Column('command_timeout_seconds', sa.Integer(), nullable=True), + sa.Column('harness_options_json', sa.Text(), nullable=True), + sa.Column('configuration_digest', sa.String(), server_default='', nullable=False), + sa.Column('is_active', sa.Boolean(), server_default='true', nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['connection_id'], ['public.provider_connection.connection_id'], ondelete='RESTRICT'), + sa.ForeignKeyConstraint(['owner_user_id'], ['public.user.user_id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['release_id'], ['public.agent_release.release_id'], ondelete='RESTRICT'), + sa.PrimaryKeyConstraint('profile_id'), + sa.UniqueConstraint('owner_user_id', 'name', name='uq_agent_profile_owner_name'), + schema='public' + ) + op.create_index('idx_agent_profile_is_active', 'agent_profile', ['is_active'], unique=False, schema='public') + op.create_index('idx_agent_profile_owner_user_id', 'agent_profile', ['owner_user_id'], unique=False, schema='public') + op.create_table('study_assignment', + sa.Column('assignment_id', sa.UUID(), nullable=False), + sa.Column('enrollment_id', sa.UUID(), nullable=False), + sa.Column('study_id', sa.UUID(), nullable=False), + sa.Column('agent_profile_id', sa.UUID(), nullable=False), + sa.Column('strategy', sa.String(), nullable=False), + sa.Column('randomization_epoch', sa.Integer(), server_default='0', nullable=False), + sa.Column('profile_digest', sa.String(), nullable=False), + sa.Column('profile_snapshot_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('assigned_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['enrollment_id'], ['public.research_enrollment.enrollment_id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['agent_profile_id'], ['public.agent_profile.profile_id'], ondelete='RESTRICT'), + sa.ForeignKeyConstraint(['study_id'], ['public.study.study_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('assignment_id'), + sa.UniqueConstraint('enrollment_id', name='uq_study_assignment_enrollment'), + schema='public' + ) + op.create_index('idx_study_assignment_enrollment_id', 'study_assignment', ['enrollment_id'], unique=False, schema='public') + op.create_index('idx_study_assignment_study_id', 'study_assignment', ['study_id'], unique=False, schema='public') + op.create_index('idx_study_assignment_agent_profile_id', 'study_assignment', ['agent_profile_id'], unique=False, schema='public') + # Participant inference budgets (shared provider key): prices, balances, + # reservation ledger and adjustment audit. Folded into the consolidated + # revision; existing databases apply the SQL in docs/research-platform/RELEASES.md. + op.create_table('provider_model_price', + sa.Column('connection_id', sa.UUID(), nullable=False), + sa.Column('model', sa.String(), nullable=False), + sa.Column('input_usd_per_million', sa.Numeric(precision=14, scale=6), nullable=False), + sa.Column('output_usd_per_million', sa.Numeric(precision=14, scale=6), nullable=False), + sa.Column('cached_input_usd_per_million', sa.Numeric(precision=14, scale=6), nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_by', sa.String(), nullable=True), + sa.CheckConstraint('input_usd_per_million >= 0', name='ck_provider_model_price_input'), + sa.CheckConstraint('output_usd_per_million >= 0', name='ck_provider_model_price_output'), + sa.CheckConstraint('cached_input_usd_per_million IS NULL OR cached_input_usd_per_million >= 0', name='ck_provider_model_price_cached_input'), + sa.ForeignKeyConstraint(['connection_id'], ['public.provider_connection.connection_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('connection_id', 'model'), + schema='public' + ) + op.create_index('idx_provider_model_price_connection', 'provider_model_price', ['connection_id'], unique=False, schema='public') + op.create_table('enrollment_inference_balance', + sa.Column('enrollment_id', sa.UUID(), nullable=False), + sa.Column('study_id', sa.UUID(), nullable=False), + sa.Column('unit', sa.String(), server_default='micro_usd', nullable=False), + sa.Column('limit_micro_usd', sa.BigInteger(), nullable=False), + sa.Column('settled_micro_usd', sa.BigInteger(), server_default='0', nullable=False), + sa.Column('reserved_micro_usd', sa.BigInteger(), server_default='0', nullable=False), + sa.Column('settled_prompt_tokens', sa.BigInteger(), server_default='0', nullable=False), + sa.Column('settled_completion_tokens', sa.BigInteger(), server_default='0', nullable=False), + sa.Column('call_count', sa.Integer(), server_default='0', nullable=False), + sa.Column('refused_count', sa.Integer(), server_default='0', nullable=False), + sa.Column('last_call_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('limit_source', sa.String(), server_default='STUDY_DEFAULT', nullable=False), + sa.Column('exhausted_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint('limit_micro_usd >= 0', name='ck_enrollment_inference_balance_limit'), + sa.CheckConstraint('settled_micro_usd >= 0', name='ck_enrollment_inference_balance_settled'), + sa.CheckConstraint('reserved_micro_usd >= 0', name='ck_enrollment_inference_balance_reserved'), + sa.ForeignKeyConstraint(['enrollment_id'], ['public.research_enrollment.enrollment_id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['study_id'], ['public.study.study_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('enrollment_id'), + schema='public' + ) + op.create_index('idx_enrollment_inference_balance_study_id', 'enrollment_inference_balance', ['study_id'], unique=False, schema='public') + op.create_table('inference_reservation', + sa.Column('reservation_id', sa.UUID(), nullable=False), + sa.Column('enrollment_id', sa.UUID(), nullable=False), + sa.Column('study_id', sa.UUID(), nullable=False), + sa.Column('connection_id', sa.UUID(), nullable=True), + sa.Column('model', sa.String(), nullable=False), + sa.Column('entry_point', sa.String(), nullable=False), + sa.Column('request_id', sa.String(), nullable=False), + sa.Column('research_session_id', sa.UUID(), nullable=True), + sa.Column('state', sa.String(), nullable=False), + sa.Column('hold_micro_usd', sa.BigInteger(), nullable=False), + sa.Column('estimated_prompt_tokens', sa.Integer(), nullable=False), + sa.Column('output_cap_tokens', sa.Integer(), nullable=False), + sa.Column('charged_micro_usd', sa.BigInteger(), nullable=True), + sa.Column('prompt_tokens', sa.Integer(), nullable=True), + sa.Column('completion_tokens', sa.Integer(), nullable=True), + sa.Column('cached_prompt_tokens', sa.Integer(), nullable=True), + sa.Column('usage_source', sa.String(), nullable=True), + sa.Column('resolution_reason', sa.String(), nullable=True), + sa.Column('upstream_status', sa.Integer(), nullable=True), + sa.Column('finish_reason', sa.String(), nullable=True), + sa.Column('reserved_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('deadline_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('resolved_at', sa.DateTime(timezone=True), nullable=True), + sa.CheckConstraint("state IN ('RESERVED', 'SETTLED', 'FORFEITED', 'VOIDED', 'EXPIRED')", name='ck_inference_reservation_state'), + sa.CheckConstraint('hold_micro_usd >= 0', name='ck_inference_reservation_hold'), + sa.CheckConstraint('charged_micro_usd IS NULL OR charged_micro_usd >= 0', name='ck_inference_reservation_charged'), + sa.ForeignKeyConstraint(['connection_id'], ['public.provider_connection.connection_id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['enrollment_id'], ['public.enrollment_inference_balance.enrollment_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('reservation_id'), + sa.UniqueConstraint('request_id', name='uq_inference_reservation_request_id'), + schema='public' + ) + op.create_index('idx_inference_reservation_enrollment_state', 'inference_reservation', ['enrollment_id', 'state'], unique=False, schema='public') + op.create_index('idx_inference_reservation_open_deadline', 'inference_reservation', ['deadline_at'], unique=False, schema='public', postgresql_where=sa.text("state = 'RESERVED'")) + op.create_index('idx_inference_reservation_study_reserved_at', 'inference_reservation', ['study_id', 'reserved_at'], unique=False, schema='public') + op.create_table('inference_budget_adjustment', + sa.Column('adjustment_id', sa.UUID(), nullable=False), + sa.Column('enrollment_id', sa.UUID(), nullable=False), + sa.Column('study_id', sa.UUID(), nullable=False), + sa.Column('kind', sa.String(), nullable=False), + sa.Column('delta_micro_usd', sa.BigInteger(), nullable=False), + sa.Column('limit_before_micro_usd', sa.BigInteger(), nullable=False), + sa.Column('limit_after_micro_usd', sa.BigInteger(), nullable=False), + sa.Column('in_flight_micro_usd', sa.BigInteger(), server_default='0', nullable=False), + sa.Column('reason', sa.String(), nullable=True), + sa.Column('actor', sa.String(), nullable=True), + sa.Column('idempotency_key', sa.String(), nullable=True), + sa.Column('request_digest', sa.String(), nullable=True), + sa.Column('occurred_at', sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint("kind IN ('TOP_UP', 'SET_LIMIT', 'APPLY_DEFAULT', 'BACKFILL')", name='ck_inference_budget_adjustment_kind'), + sa.ForeignKeyConstraint(['enrollment_id'], ['public.enrollment_inference_balance.enrollment_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('adjustment_id'), + sa.UniqueConstraint('study_id', 'idempotency_key', name='uq_inference_budget_adjustment_idempotency'), + schema='public' + ) + op.create_index('idx_inference_budget_adjustment_enrollment', 'inference_budget_adjustment', ['enrollment_id', 'occurred_at'], unique=False, schema='public') + op.create_index('idx_inference_budget_adjustment_study', 'inference_budget_adjustment', ['study_id', 'occurred_at'], unique=False, schema='public') + op.create_table('telemetry_batch_receipt', + sa.Column('receipt_id', sa.UUID(), nullable=False), + sa.Column('batch_id', sa.String(), nullable=False), + sa.Column('enrollment_id', sa.UUID(), nullable=True), + sa.Column('research_session_id', sa.UUID(), nullable=True), + sa.Column('accepted_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('receipt_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.ForeignKeyConstraint(['enrollment_id'], ['public.research_enrollment.enrollment_id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['research_session_id'], ['public.research_session.session_id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('receipt_id'), + sa.UniqueConstraint('batch_id'), + schema='public' + ) + op.create_index('idx_telemetry_batch_receipt_session_id', 'telemetry_batch_receipt', ['research_session_id'], unique=False, schema='public') + op.create_table('agent_task', + sa.Column('task_id', sa.UUID(), nullable=False), + sa.Column('session_id', sa.UUID(), nullable=True), + sa.Column('source', sa.String(), server_default='plugin', nullable=False), + sa.Column('owner_user_id', sa.UUID(), nullable=True), + sa.Column('funding_owner_user_id', sa.UUID(), nullable=True), + sa.Column('owner_project_id', sa.UUID(), nullable=True), + sa.Column('external_run_id', sa.String(), nullable=True), + sa.Column('agent_session_id', sa.String(), nullable=True), + sa.Column('study_id', sa.UUID(), nullable=True), + sa.Column('study_assignment_id', sa.UUID(), nullable=True), + sa.Column('research_session_id', sa.UUID(), nullable=True), + sa.Column('enrollment_id', sa.UUID(), nullable=True), + sa.Column('profile_id', sa.UUID(), nullable=True), + sa.Column('agent_profile', sa.String(), nullable=False), + sa.Column('model', sa.String(), nullable=False), + sa.Column('temperature', sa.Double(), nullable=True), + sa.Column('approval_policy', sa.String(), nullable=False), + sa.Column('tools_json', sa.Text(), nullable=False), + sa.Column('framework_version', sa.String(), nullable=True), + sa.Column('policy_snapshot', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('observed_framework_version', sa.String(), nullable=True), + sa.Column('observed_tools_json', sa.Text(), nullable=True), + sa.Column('consent_content_storage', sa.Boolean(), nullable=True), + sa.Column('next_event_index', sa.Integer(), server_default='0', nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('started_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('completed_at', sa.DateTime(), nullable=True), + sa.Column('total_steps', sa.Integer(), nullable=True), + sa.Column('input_tokens', sa.Integer(), nullable=True), + sa.Column('output_tokens', sa.Integer(), nullable=True), + sa.Column('latest_event_id', sa.UUID(), nullable=True), + sa.Column('task_description', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['funding_owner_user_id'], ['public.user.user_id'], ondelete='SET NULL'), + # The cyclic latest_event_id FK is added explicitly after both tables exist + # (declared use_alter=True in the model, so it is not created inline here). + sa.ForeignKeyConstraint(['owner_user_id'], ['public.user.user_id'], ), + sa.ForeignKeyConstraint(['profile_id'], ['public.agent_profile.profile_id'], ), + sa.ForeignKeyConstraint(['session_id'], ['public.session.session_id'], ), + sa.ForeignKeyConstraint(['study_assignment_id'], ['public.study_assignment.assignment_id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['study_id'], ['public.study.study_id'], ), + sa.PrimaryKeyConstraint('task_id'), + sa.UniqueConstraint('external_run_id'), + schema='public' + ) + op.create_index('idx_agent_task_enrollment_id', 'agent_task', ['enrollment_id'], unique=False, schema='public') + op.create_index('idx_agent_task_owner_user_id', 'agent_task', ['owner_user_id'], unique=False, schema='public') + op.create_index('idx_agent_task_profile_id', 'agent_task', ['profile_id'], unique=False, schema='public') + op.create_index('idx_agent_task_research_session_id', 'agent_task', ['research_session_id'], unique=False, schema='public') + op.create_index('idx_agent_task_session_id', 'agent_task', ['session_id'], unique=False, schema='public') + op.create_index('idx_agent_task_status', 'agent_task', ['status'], unique=False, schema='public') + op.create_index('idx_agent_task_study_assignment_id', 'agent_task', ['study_assignment_id'], unique=False, schema='public') + op.create_index('idx_agent_task_study_id', 'agent_task', ['study_id'], unique=False, schema='public') + op.create_table('agent_edit', + sa.Column('edit_id', sa.UUID(), nullable=False), + sa.Column('task_id', sa.UUID(), nullable=False), + sa.Column('file_path', sa.Text(), nullable=False), + sa.Column('was_accepted', sa.Boolean(), nullable=True), + sa.Column('was_modified', sa.Boolean(), nullable=True), + sa.Column('decided_at', sa.DateTime(), nullable=True), + sa.Column('diff_text', sa.Text(), nullable=True), + sa.Column('edit_delta_json', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['task_id'], ['public.agent_task.task_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('edit_id'), + schema='public' + ) + op.create_index('idx_agent_edit_task_id', 'agent_edit', ['task_id'], unique=False, schema='public') + op.create_table('agent_event', + sa.Column('event_id', sa.UUID(), nullable=False), + sa.Column('task_id', sa.UUID(), nullable=False), + sa.Column('event_index', sa.Integer(), nullable=False), + sa.Column('event_type', sa.String(), nullable=False), + sa.Column('source', sa.String(), nullable=True), + sa.Column('source_event_id', sa.String(), nullable=True), + sa.Column('schema_version', sa.String(), nullable=True), + sa.Column('latency_ms', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('occurred_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('span_id', sa.String(), nullable=True), + sa.Column('parent_span_id', sa.UUID(), nullable=True), + sa.Column('request_id', sa.String(), nullable=True), + sa.Column('chat_session_index', sa.Integer(), nullable=True), + sa.Column('model', sa.String(), nullable=True), + sa.Column('agent_profile', sa.String(), nullable=True), + sa.Column('streaming', sa.Boolean(), nullable=True), + sa.Column('message_count', sa.Integer(), nullable=True), + sa.Column('role_system_count', sa.Integer(), nullable=True), + sa.Column('role_user_count', sa.Integer(), nullable=True), + sa.Column('role_assistant_count', sa.Integer(), nullable=True), + sa.Column('role_tool_count', sa.Integer(), nullable=True), + sa.Column('tools_kept', sa.Integer(), nullable=True), + sa.Column('tools_stripped', sa.Integer(), nullable=True), + sa.Column('tool_names_requested', sa.ARRAY(sa.String()), nullable=True), + sa.Column('max_tokens', sa.Integer(), nullable=True), + sa.Column('prompt_tokens', sa.Integer(), nullable=True), + sa.Column('completion_tokens', sa.Integer(), nullable=True), + sa.Column('total_tokens', sa.Integer(), nullable=True), + sa.Column('finish_reason', sa.String(), nullable=True), + sa.Column('upstream_status', sa.Integer(), nullable=True), + sa.Column('step_index', sa.Integer(), nullable=True), + sa.Column('context_window_size_bytes', sa.Integer(), nullable=True), + sa.Column('active_file', sa.Text(), nullable=True), + sa.Column('first_message_hash', sa.String(), nullable=True), + sa.Column('chat_new_session_detected', sa.Boolean(), nullable=True), + sa.Column('experiment_tool_access_enabled', sa.Boolean(), nullable=True), + sa.Column('experiment_approval_policy', sa.String(), nullable=True), + sa.Column('first_system_message', sa.Text(), nullable=True), + sa.Column('last_user_message', sa.Text(), nullable=True), + sa.Column('response_text', sa.Text(), nullable=True), + sa.Column('tool_name', sa.String(), nullable=True), + sa.Column('tool_arguments_length', sa.Integer(), nullable=True), + sa.Column('tool_result_length', sa.Integer(), nullable=True), + sa.Column('tool_arguments', sa.Text(), nullable=True), + sa.Column('tool_result', sa.Text(), nullable=True), + sa.Column('extra_json', sa.Text(), nullable=True), + sa.Column('payload_json', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['task_id'], ['public.agent_task.task_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('event_id'), + sa.UniqueConstraint('task_id', 'event_index', name='uq_agent_event_task_event_index'), + sa.UniqueConstraint('task_id', 'source', 'source_event_id', name='uq_agent_event_source_identity'), + schema='public' + ) + op.create_index('idx_agent_event_created_at', 'agent_event', ['created_at'], unique=False, schema='public') + op.create_index('idx_agent_event_event_type', 'agent_event', ['event_type'], unique=False, schema='public') + op.create_index('idx_agent_event_task_id', 'agent_event', ['task_id'], unique=False, schema='public') + op.add_column('study', sa.Column('is_research', sa.Boolean(), server_default='false', nullable=False), schema='public') + op.add_column('study', sa.Column('research_status', sa.String(), nullable=True), schema='public') + op.add_column('study', sa.Column('research_config_json', postgresql.JSONB(astext_type=sa.Text()), nullable=True), schema='public') + op.add_column('study', sa.Column('research_config_digest', sa.String(), nullable=True), schema='public') + op.add_column('study', sa.Column('join_code', sa.String(), nullable=True), schema='public') + op.add_column('study', sa.Column('consent_locked_at', sa.DateTime(timezone=True), nullable=True), schema='public') + op.add_column('study', sa.Column('stopped_at', sa.DateTime(timezone=True), nullable=True), schema='public') + op.add_column('study', sa.Column('stopped_by', sa.String(), nullable=True), schema='public') + op.add_column('study', sa.Column('inference_budget_default_micro_usd', sa.BigInteger(), server_default='0', nullable=False), schema='public') + op.add_column('study', sa.Column('inference_budget_warning_fraction', sa.Numeric(precision=4, scale=3), server_default='0.800', nullable=False), schema='public') + op.add_column('study', sa.Column('inference_budget_updated_at', sa.DateTime(timezone=True), nullable=True), schema='public') + op.add_column('study', sa.Column('inference_budget_updated_by', sa.String(), nullable=True), schema='public') + op.create_table('study_agent_profile', + sa.Column('study_id', sa.UUID(), nullable=False), + sa.Column('profile_id', sa.UUID(), nullable=False), + sa.Column('profile_digest', sa.String(), nullable=False), + sa.Column('profile_snapshot_json', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('selection_order', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['profile_id'], ['public.agent_profile.profile_id'], ondelete='RESTRICT'), + sa.ForeignKeyConstraint(['study_id'], ['public.study.study_id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('study_id', 'profile_id'), + sa.UniqueConstraint('study_id', 'profile_id', name='uq_study_agent_profile_selection'), + schema='public' + ) + op.create_index('idx_study_agent_profile_profile_id', 'study_agent_profile', ['profile_id'], unique=False, schema='public') + op.alter_column('study', 'default_config_id', + existing_type=sa.BIGINT(), + nullable=True, + schema='public') + op.create_index('idx_study_research_status', 'study', ['research_status'], unique=False, schema='public') + op.create_index('uq_study_research_join_code', 'study', ['join_code'], unique=True, schema='public', postgresql_where=sa.text('is_research AND join_code IS NOT NULL')) + op.add_column('user', sa.Column('can_research', sa.Boolean(), server_default='false', nullable=False), schema='public') + op.create_index('idx_user_can_research', 'user', ['can_research'], unique=False, schema='public') + op.add_column('user', sa.Column('data_collection_opted_out_at', sa.DateTime(timezone=True), nullable=True), schema='public') + # Cyclic FK (agent_task.latest_event_id -> agent_event.event_id) declared + # with use_alter=True in the model, so autogenerate skips it. Added by hand + # after both tables exist. + # Research referential integrity (M02). Ownership/enrollment/study are + # real relationships; telemetry uses SET NULL so retention policy, not a + # parent-row delete, decides when events disappear. + op.create_foreign_key('fk_research_participant_account', 'research_participant', 'user', ['account_id'], ['user_id'], source_schema='public', referent_schema='public', ondelete='CASCADE') + op.create_foreign_key('fk_research_session_enrollment', 'research_session', 'research_enrollment', ['enrollment_id'], ['enrollment_id'], source_schema='public', referent_schema='public', ondelete='CASCADE') + op.create_foreign_key('fk_research_session_study', 'research_session', 'study', ['study_id'], ['study_id'], source_schema='public', referent_schema='public', ondelete='CASCADE') + op.create_foreign_key('fk_research_event_study', 'research_event', 'study', ['study_id'], ['study_id'], source_schema='public', referent_schema='public', ondelete='SET NULL') + op.create_foreign_key('fk_research_event_enrollment', 'research_event', 'research_enrollment', ['enrollment_id'], ['enrollment_id'], source_schema='public', referent_schema='public', ondelete='SET NULL') + op.create_foreign_key('fk_research_event_session', 'research_event', 'research_session', ['research_session_id'], ['session_id'], source_schema='public', referent_schema='public', ondelete='SET NULL') + op.create_foreign_key( + 'agent_task_latest_event_id_fkey', + 'agent_task', + 'agent_event', + ['latest_event_id'], + ['event_id'], + source_schema='public', + referent_schema='public', + ) + # Provider-backed classic models have no confidence (NULL), matching init.sql. + op.alter_column('had_generation', 'confidence', existing_type=sa.Double(), + nullable=True, schema='public') + # ### end Alembic commands ### + + +def downgrade() -> None: + for _name, _table in ( + ('fk_research_event_session', 'research_event'), + ('fk_research_event_enrollment', 'research_event'), + ('fk_research_event_study', 'research_event'), + ('fk_research_session_study', 'research_session'), + ('fk_research_session_enrollment', 'research_session'), + ('fk_research_participant_account', 'research_participant'), + ): + op.drop_constraint(_name, _table, schema='public', type_='foreignkey') + """Downgrade database schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('user', 'data_collection_opted_out_at', schema='public') + op.drop_index('idx_user_can_research', table_name='user', schema='public') + op.drop_column('user', 'can_research', schema='public') + op.drop_index('uq_study_research_join_code', table_name='study', schema='public', postgresql_where=sa.text('is_research AND join_code IS NOT NULL')) + op.drop_index('idx_study_research_status', table_name='study', schema='public') + op.drop_index('idx_study_agent_profile_profile_id', table_name='study_agent_profile', schema='public') + op.drop_table('study_agent_profile', schema='public') + op.drop_column('study', 'inference_budget_updated_by', schema='public') + op.drop_column('study', 'inference_budget_updated_at', schema='public') + op.drop_column('study', 'inference_budget_warning_fraction', schema='public') + op.drop_column('study', 'inference_budget_default_micro_usd', schema='public') + op.drop_column('study', 'stopped_by', schema='public') + op.drop_column('study', 'stopped_at', schema='public') + op.drop_column('study', 'consent_locked_at', schema='public') + op.drop_column('study', 'join_code', schema='public') + op.drop_column('study', 'research_config_digest', schema='public') + op.drop_column('study', 'research_config_json', schema='public') + op.drop_column('study', 'research_status', schema='public') + op.alter_column('study', 'default_config_id', + existing_type=sa.BIGINT(), + nullable=False, + schema='public') + op.drop_column('study', 'is_research', schema='public') + op.drop_index('idx_agent_event_task_id', table_name='agent_event', schema='public') + op.drop_index('idx_agent_event_event_type', table_name='agent_event', schema='public') + op.drop_index('idx_agent_event_created_at', table_name='agent_event', schema='public') + op.drop_constraint('agent_task_latest_event_id_fkey', 'agent_task', schema='public', type_='foreignkey') + op.drop_table('agent_event', schema='public') + op.drop_index('idx_agent_edit_task_id', table_name='agent_edit', schema='public') + op.drop_table('agent_edit', schema='public') + op.drop_index('idx_agent_task_study_id', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_study_assignment_id', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_status', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_session_id', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_research_session_id', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_profile_id', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_owner_user_id', table_name='agent_task', schema='public') + op.drop_index('idx_agent_task_enrollment_id', table_name='agent_task', schema='public') + op.drop_table('agent_task', schema='public') + op.drop_index('idx_telemetry_batch_receipt_session_id', table_name='telemetry_batch_receipt', schema='public') + op.drop_table('telemetry_batch_receipt', schema='public') + op.drop_index('idx_inference_budget_adjustment_study', table_name='inference_budget_adjustment', schema='public') + op.drop_index('idx_inference_budget_adjustment_enrollment', table_name='inference_budget_adjustment', schema='public') + op.drop_table('inference_budget_adjustment', schema='public') + op.drop_index('idx_inference_reservation_study_reserved_at', table_name='inference_reservation', schema='public') + op.drop_index('idx_inference_reservation_open_deadline', table_name='inference_reservation', schema='public', postgresql_where=sa.text("state = 'RESERVED'")) + op.drop_index('idx_inference_reservation_enrollment_state', table_name='inference_reservation', schema='public') + op.drop_table('inference_reservation', schema='public') + op.drop_index('idx_enrollment_inference_balance_study_id', table_name='enrollment_inference_balance', schema='public') + op.drop_table('enrollment_inference_balance', schema='public') + op.drop_index('idx_provider_model_price_connection', table_name='provider_model_price', schema='public') + op.drop_table('provider_model_price', schema='public') + op.drop_index('idx_study_assignment_agent_profile_id', table_name='study_assignment', schema='public') + op.drop_index('idx_study_assignment_study_id', table_name='study_assignment', schema='public') + op.drop_index('idx_study_assignment_enrollment_id', table_name='study_assignment', schema='public') + op.drop_table('study_assignment', schema='public') + op.drop_index('idx_agent_profile_owner_user_id', table_name='agent_profile', schema='public') + op.drop_index('idx_agent_profile_is_active', table_name='agent_profile', schema='public') + op.drop_table('agent_profile', schema='public') + op.drop_index('uq_research_enrollment_active_participant', table_name='research_enrollment', schema='public', postgresql_where=sa.text("status = 'ACTIVE'")) + op.drop_index('idx_research_enrollment_study_id', table_name='research_enrollment', schema='public') + op.drop_index('idx_research_enrollment_status', table_name='research_enrollment', schema='public') + op.drop_index('idx_research_enrollment_participant_id', table_name='research_enrollment', schema='public') + op.drop_index('idx_research_enrollment_participant_code', table_name='research_enrollment', schema='public') + op.drop_table('research_enrollment', schema='public') + op.drop_index('idx_research_agent_run_session_id', table_name='research_agent_run', schema='public') + op.drop_table('research_agent_run', schema='public') + op.drop_index('uq_research_session_active_context', table_name='research_session', schema='public', postgresql_where=sa.text("state NOT IN ('ended', 'revoked')")) + op.drop_index('idx_research_session_study_id', table_name='research_session', schema='public') + op.drop_index('idx_research_session_state', table_name='research_session', schema='public') + op.drop_index('idx_research_session_enrollment_id', table_name='research_session', schema='public') + op.drop_index('idx_research_session_context_id', table_name='research_session', schema='public') + op.drop_table('research_session', schema='public') + op.drop_index('idx_research_retention_job_state', table_name='research_retention_job', schema='public') + op.drop_index('idx_research_retention_job_enrollment', table_name='research_retention_job', schema='public') + op.drop_table('research_retention_job', schema='public') + op.drop_index('idx_research_record_study_id', table_name='research_record', schema='public') + op.drop_index('idx_research_record_scope', table_name='research_record', schema='public') + op.drop_index('idx_research_record_occurred_at', table_name='research_record', schema='public') + op.drop_index('idx_research_record_kind', table_name='research_record', schema='public') + op.drop_table('research_record', schema='public') + op.drop_index('idx_research_participant_account_id', table_name='research_participant', schema='public') + op.drop_table('research_participant', schema='public') + op.drop_index('idx_research_event_session_id', table_name='research_event', schema='public') + op.drop_index('idx_research_event_occurred_at', table_name='research_event', schema='public') + op.drop_index('idx_research_event_event_type', table_name='research_event', schema='public') + op.drop_index('idx_research_event_enrollment_id', table_name='research_event', schema='public') + op.drop_index('idx_research_event_emitter_id', table_name='research_event', schema='public') + op.drop_index('idx_research_event_digest', table_name='research_event', schema='public') + op.drop_index('idx_research_event_retention_state', table_name='research_event', schema='public') + op.drop_index('idx_research_event_agent_run_id', table_name='research_event', schema='public') + op.drop_index('idx_research_event_study_id', table_name='research_event', schema='public') + op.drop_table('research_event', schema='public') + op.drop_index('idx_provider_connection_is_active', table_name='provider_connection', schema='public') + op.drop_table('provider_connection', schema='public') + op.drop_index('idx_agent_release_status', table_name='agent_release', schema='public') + op.drop_index('idx_agent_release_agent_id', table_name='agent_release', schema='public') + op.drop_table('agent_release', schema='public') + op.drop_index('idx_agent_memory_updated_at', table_name='agent_memory', schema='public') + op.drop_index('idx_agent_memory_owner_user_id', table_name='agent_memory', schema='public') + op.drop_index('idx_agent_memory_owner_project_id', table_name='agent_memory', schema='public') + op.drop_table('agent_memory', schema='public') + op.drop_index('idx_acp_capability_receipt_status', table_name='acp_capability_receipt', schema='public') + op.drop_index('idx_acp_capability_receipt_captured_at', table_name='acp_capability_receipt', schema='public') + op.drop_index('idx_acp_capability_receipt_agent', table_name='acp_capability_receipt', schema='public') + op.drop_table('acp_capability_receipt', schema='public') + # ### end Alembic commands ### \ No newline at end of file diff --git a/src/database/migration/versions/b7c1d2e3f4a5_drop_emitter_sequence_unique.py b/src/database/migration/versions/b7c1d2e3f4a5_drop_emitter_sequence_unique.py new file mode 100644 index 0000000..5d2c2ca --- /dev/null +++ b/src/database/migration/versions/b7c1d2e3f4a5_drop_emitter_sequence_unique.py @@ -0,0 +1,41 @@ +"""drop the (session, emitter, sequence) unique constraint on research_event + +Revision ID: b7c1d2e3f4a5 +Revises: 8a0084080b46 +Create Date: 2026-09-27 02:10:00 + +An emitter whose sequence counter restarts inside a live session (an IDE +restart, a re-launched proxy) reused ``(research_session_id, emitter_id, +emitter_sequence)`` and the unique constraint made ingestion reject - and the +client delete - the later facts (production-readiness review C-01). Identity is +``event_id`` + digest; the composite stays indexed for ordering and gap +diagnostics only. +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'b7c1d2e3f4a5' +down_revision = '8a0084080b46' +branch_labels = None +depends_on = None + +_CONSTRAINT = "uq_research_event_session_emitter_sequence" +_INDEX = "idx_research_event_session_emitter_sequence" + + +def upgrade() -> None: + op.execute(f"ALTER TABLE public.research_event DROP CONSTRAINT IF EXISTS {_CONSTRAINT}") + op.execute( + f"CREATE INDEX IF NOT EXISTS {_INDEX} ON public.research_event " + "(research_session_id, emitter_id, emitter_sequence)" + ) + + +def downgrade() -> None: + # Re-creating the unique constraint fails if overlapping sequences were + # stored in the meantime; that is the intended signal (the data is valid). + op.execute(f"DROP INDEX IF EXISTS public.{_INDEX}") + op.execute( + f"ALTER TABLE public.research_event ADD CONSTRAINT {_CONSTRAINT} " + "UNIQUE (research_session_id, emitter_id, emitter_sequence)" + ) diff --git a/src/database/migration/versions/c5e8f1a2d3b4_enrollment_consent_version.py b/src/database/migration/versions/c5e8f1a2d3b4_enrollment_consent_version.py new file mode 100644 index 0000000..3975a99 --- /dev/null +++ b/src/database/migration/versions/c5e8f1a2d3b4_enrollment_consent_version.py @@ -0,0 +1,34 @@ +"""record which consent version each enrollment accepted + +Revision ID: c5e8f1a2d3b4 +Revises: b7c1d2e3f4a5 +Create Date: 2026-10-04 03:30:00 + +A study may carry its own consent document and tick-box statements; each +enrollment now stores the digest of the consent view it accepted and that view +with the participant's answers. Both columns are nullable and older code +ignores them; enrollments created before this revision keep null values. +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'c5e8f1a2d3b4' +down_revision = 'b7c1d2e3f4a5' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + "ALTER TABLE public.research_enrollment " + "ADD COLUMN IF NOT EXISTS consent_digest VARCHAR NULL, " + "ADD COLUMN IF NOT EXISTS consent_snapshot_json JSONB NULL" + ) + + +def downgrade() -> None: + op.execute( + "ALTER TABLE public.research_enrollment " + "DROP COLUMN IF EXISTS consent_snapshot_json, " + "DROP COLUMN IF EXISTS consent_digest" + ) diff --git a/src/database/migration/versions/d41e7c9a2b6f_agent_profile_commands_denylist.py b/src/database/migration/versions/d41e7c9a2b6f_agent_profile_commands_denylist.py new file mode 100644 index 0000000..04b0111 --- /dev/null +++ b/src/database/migration/versions/d41e7c9a2b6f_agent_profile_commands_denylist.py @@ -0,0 +1,32 @@ +"""store the commands a built-in agent profile blocks + +Revision ID: d41e7c9a2b6f +Revises: c5e8f1a2d3b4 +Create Date: 2026-10-10 12:00:00 + +Built-in (``code4me2-agent``) profiles now list the programs the agent may not +run (``commands_denylist``) instead of the ones it may run. The new column is +nullable (NULL blocks nothing) and older code ignores it. The retired +``commands_allowlist_json`` column is left in place and unused, so a rollback +to the previous backend still finds it; study snapshots keep their own copy. +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'd41e7c9a2b6f' +down_revision = 'c5e8f1a2d3b4' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + "ALTER TABLE public.agent_profile " + "ADD COLUMN IF NOT EXISTS commands_denylist_json TEXT NULL" + ) + + +def downgrade() -> None: + op.execute( + "ALTER TABLE public.agent_profile DROP COLUMN IF EXISTS commands_denylist_json" + ) diff --git a/src/database/research_schemas.py b/src/database/research_schemas.py new file mode 100644 index 0000000..ecb2574 --- /dev/null +++ b/src/database/research_schemas.py @@ -0,0 +1,774 @@ +"""SQLAlchemy ORM models for the research platform stores. + +The research platform keeps its own namespace so operational agent tables stay +independent. Data that is a pure projection of an existing +JSON document lives inside that document rather than in a child table: + +* capability evidence is ``acp_capability_receipt.receipt_json.evidence[]``; +* agent artifacts/adapter are ``agent_release.release_json.artifacts[]``/``.adapter``; +* runtime packaging evidence is ``agent_release.release_json.package_json``; +* release qualification comes from ``agent_release.release_json.tests``; +* a capability snapshot is ``research_agent_run.snapshot_json``; +* selected profiles are ``study_agent_profile.profile_snapshot_json``; +* study identity/configuration is owned by ``study``; +* draft lifecycle state is ``study.research_status = DRAFT``; +* an emitter cursor is ``max(research_event.emitter_sequence)``; +* rejections are ``telemetry_batch_receipt.receipt_json.rejected[]``; +* session transitions are appended to ``research_session.transitions_json``; +* a withdrawal is ``research_enrollment.withdrawal_json``; +* retention evidence is ``research_retention_job.evidence_json``. + +Two cross-cutting concerns get a generic table: ``research_record`` for every +append-only audit/evidence family (kill switch, health, release evidence, pilot +run, export, retention). Coverage/metric projections are derived +purely from canonical events and are not persisted. +""" + +from datetime import datetime + +from sqlalchemy import ( + BigInteger, + CheckConstraint, + Column, + DateTime, + ForeignKey, + Index, + Integer, + String, + UniqueConstraint, + text, +) +from sqlalchemy.dialects.postgresql import JSONB, UUID + +from .db import Base + + +# NOTE: ``os`` is used as a column attribute name because it is part of the +# published environment tuple. It shadows the stdlib module only inside this +# class scope, which is safe because the module never imports ``os``. +class AcpCapabilityReceipt(Base): + """One version-pinned, redacted capability receipt.""" + + __tablename__ = "acp_capability_receipt" + __table_args__ = ( + Index("idx_acp_capability_receipt_agent", "agent_id", "agent_version"), + Index("idx_acp_capability_receipt_status", "status"), + Index("idx_acp_capability_receipt_captured_at", "captured_at"), + {"schema": "public"}, + ) + + receipt_id = Column(UUID(as_uuid=True), primary_key=True) + captured_at = Column(DateTime(timezone=True), nullable=False) + ide_build = Column(String, nullable=False) + ai_assistant_build = Column(String, nullable=False) + plugin_version = Column(String, nullable=False) + os = Column(String, nullable=False) + arch = Column(String, nullable=False) + agent_id = Column(String, nullable=False) + agent_version = Column(String, nullable=False) + adapter_version = Column(String, nullable=True) + protocol_version = Column(String, nullable=False) + status = Column(String, nullable=False) + # Unique: a receipt's canonical content is its identity, so storing the same + # bytes twice is a duplicate rather than a new observation. + content_hash = Column(String, nullable=False, unique=True) + receipt_json = Column(JSONB, nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + +# --------------------------------------------------------------------------- +# Issue 02: versioned study protocol and publication +# --------------------------------------------------------------------------- + + +class StudyAgentProfile(Base): + """One profile selected by a study at creation time. + + The digest and non-secret snapshot keep a DRAFT study stable if the shared + profile is edited before the study receives its first consent. Assignment + rows take their own snapshot when a participant is enrolled. + """ + + __tablename__ = "study_agent_profile" + __table_args__ = ( + Index("idx_study_agent_profile_profile_id", "profile_id"), + UniqueConstraint( + "study_id", "profile_id", name="uq_study_agent_profile_selection" + ), + {"schema": "public"}, + ) + + study_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study.study_id", ondelete="CASCADE"), + primary_key=True, + ) + profile_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_profile.profile_id", ondelete="RESTRICT"), + primary_key=True, + ) + profile_digest = Column(String, nullable=False) + profile_snapshot_json = Column(JSONB, nullable=False) + selection_order = Column(Integer, nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + +# --------------------------------------------------------------------------- +# Issue 04: agent registry and capability contract +# --------------------------------------------------------------------------- + + +class AgentRelease(Base): + """One digest-pinned agent release record. + + Identity is ``(agent_id, release_id, source_manifest_digest)``: a changed + upstream manifest digest is a distinct release, not an in-place update. The + unique constraint on ``(agent_id, source_manifest_digest)`` additionally + makes the same source bytes impossible to register twice. + """ + + __tablename__ = "agent_release" + __table_args__ = ( + Index("idx_agent_release_agent_id", "agent_id"), + Index("idx_agent_release_status", "status"), + UniqueConstraint( + "agent_id", + "source_manifest_digest", + name="uq_agent_release_agent_manifest", + ), + {"schema": "public"}, + ) + + release_id = Column(String, primary_key=True) + agent_id = Column(String, nullable=False) + source_manifest_digest = Column(String, nullable=False) + status = Column(String, nullable=False) + release_json = Column(JSONB, nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + +# --------------------------------------------------------------------------- +# Issue 03: participant identity, consent and withdrawal +# --------------------------------------------------------------------------- + + +class ResearchParticipant(Base): + """Private account-to-participant mapping. + + This is the only research table that links a login account to a study + participant. It is privileged and is never returned to researchers or + embedded in an export. + """ + + __tablename__ = "research_participant" + __table_args__ = ( + Index("idx_research_participant_account_id", "account_id"), + {"schema": "public"}, + ) + + participant_id = Column(UUID(as_uuid=True), primary_key=True) + account_id = Column( + UUID(as_uuid=True), + ForeignKey("public.user.user_id", ondelete="CASCADE"), + nullable=False, + unique=True, + ) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + +class ResearchEnrollment(Base): + """One participant's study enrollment and consent identity.""" + + __tablename__ = "research_enrollment" + __table_args__ = ( + Index("idx_research_enrollment_participant_id", "participant_id"), + Index("idx_research_enrollment_study_id", "study_id"), + Index("idx_research_enrollment_participant_code", "participant_code"), + Index("idx_research_enrollment_status", "status"), + UniqueConstraint( + "participant_id", + "study_id", + name="uq_research_enrollment_participant_study", + ), + # One active enrollment per account across the whole platform. ACTIVE is + # the only state that holds the slot; a completed study frees it. + Index( + "uq_research_enrollment_active_participant", + "participant_id", + unique=True, + postgresql_where=text("status = 'ACTIVE'"), + ), + {"schema": "public"}, + ) + + enrollment_id = Column(UUID(as_uuid=True), primary_key=True) + participant_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_participant.participant_id", ondelete="CASCADE"), + nullable=False, + ) + study_id = Column(UUID(as_uuid=True), nullable=False) + # Random study-local pseudonym. Never derived from the account id. + participant_code = Column(String, nullable=False) + status = Column(String, nullable=False) + revocation_epoch = Column(Integer, nullable=False, server_default="0", default=0) + eligibility_json = Column(JSONB, nullable=False) + enrolled_at = Column(DateTime(timezone=True), nullable=False) + updated_at = Column(DateTime(timezone=True), nullable=False) + # The single consent acceptance: set once when the participant joins. There + # is no re-consent or withdrawal. + consent_accepted_at = Column(DateTime(timezone=True), nullable=True) + # What was accepted: the digest of the consent view (document, platform + # notice, statements) and that view with the participant's answers. Null for + # enrollments created before consent versions were recorded. + consent_digest = Column(String, nullable=True) + consent_snapshot_json = Column(JSONB, nullable=True) + retention_action = Column(String, nullable=False) + + +# --------------------------------------------------------------------------- +# Issue 05: enrollment assignment and bootstrap capabilities +# --------------------------------------------------------------------------- + + +class StudyAssignment(Base): + """The immutable, sticky agent profile assigned to one enrollment.""" + + __tablename__ = "study_assignment" + __table_args__ = ( + Index("idx_study_assignment_enrollment_id", "enrollment_id"), + Index("idx_study_assignment_study_id", "study_id"), + Index("idx_study_assignment_agent_profile_id", "agent_profile_id"), + UniqueConstraint("enrollment_id", name="uq_study_assignment_enrollment"), + {"schema": "public"}, + ) + + assignment_id = Column(UUID(as_uuid=True), primary_key=True) + enrollment_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_enrollment.enrollment_id", ondelete="CASCADE"), + nullable=False, + ) + study_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study.study_id", ondelete="CASCADE"), + nullable=False, + ) + agent_profile_id = Column( + UUID(as_uuid=True), + ForeignKey("public.agent_profile.profile_id", ondelete="RESTRICT"), + nullable=False, + ) + strategy = Column(String, nullable=False, default="RANDOM_EQUAL") + randomization_epoch = Column(Integer, nullable=False, server_default="0", default=0) + profile_digest = Column(String, nullable=False) + profile_snapshot_json = Column(JSONB, nullable=False) + status = Column(String, nullable=False, default="ACTIVE") + assigned_at = Column(DateTime(timezone=True), nullable=False) + + +# --------------------------------------------------------------------------- +# Participant inference budgets (shared provider key) +# --------------------------------------------------------------------------- + +#: Unit stored on a balance row. Amounts are unit-agnostic integers; only the +#: pricing layer knows about USD, which keeps a later switch to tokens local. +BUDGET_UNIT_MICRO_USD = "micro_usd" +#: ``limit_source`` values for :class:`EnrollmentInferenceBalance`. +LIMIT_SOURCE_STUDY_DEFAULT = "STUDY_DEFAULT" +LIMIT_SOURCE_ADJUSTED = "ADJUSTED" +LIMIT_SOURCE_BACKFILL = "BACKFILL" +#: ``state`` values for :class:`InferenceReservation`. +RESERVATION_RESERVED = "RESERVED" +RESERVATION_SETTLED = "SETTLED" +RESERVATION_FORFEITED = "FORFEITED" +RESERVATION_VOIDED = "VOIDED" +RESERVATION_EXPIRED = "EXPIRED" +RESERVATION_STATES = ( + RESERVATION_RESERVED, + RESERVATION_SETTLED, + RESERVATION_FORFEITED, + RESERVATION_VOIDED, + RESERVATION_EXPIRED, +) +#: ``kind`` values for :class:`InferenceBudgetAdjustment`. +ADJUSTMENT_TOP_UP = "TOP_UP" +ADJUSTMENT_SET_LIMIT = "SET_LIMIT" +ADJUSTMENT_APPLY_DEFAULT = "APPLY_DEFAULT" +ADJUSTMENT_BACKFILL = "BACKFILL" +ADJUSTMENT_KINDS = ( + ADJUSTMENT_TOP_UP, + ADJUSTMENT_SET_LIMIT, + ADJUSTMENT_APPLY_DEFAULT, + ADJUSTMENT_BACKFILL, +) + + +class EnrollmentInferenceBalance(Base): + """One participant's inference budget: the row every metered call reserves on. + + ``available = limit - settled - reserved``. A call is forwarded only after + ``UPDATE ... SET reserved = reserved + hold WHERE available >= hold`` succeeds + (see ``research.budget.ledger.reserve``); that statement, not a CHECK + constraint, is the limit. ``settled`` may exceed ``limit`` after a + settlement that overshot its hold or after a limit was lowered; the next + reservation is then refused. ``exhausted_at`` is a display projection. + """ + + __tablename__ = "enrollment_inference_balance" + __table_args__ = ( + CheckConstraint("limit_micro_usd >= 0", name="ck_enrollment_inference_balance_limit"), + CheckConstraint("settled_micro_usd >= 0", name="ck_enrollment_inference_balance_settled"), + CheckConstraint("reserved_micro_usd >= 0", name="ck_enrollment_inference_balance_reserved"), + Index("idx_enrollment_inference_balance_study_id", "study_id"), + {"schema": "public"}, + ) + + enrollment_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_enrollment.enrollment_id", ondelete="CASCADE"), + primary_key=True, + ) + study_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study.study_id", ondelete="CASCADE"), + nullable=False, + ) + unit = Column( + String, nullable=False, server_default=BUDGET_UNIT_MICRO_USD, default=BUDGET_UNIT_MICRO_USD + ) + limit_micro_usd = Column(BigInteger, nullable=False) + settled_micro_usd = Column(BigInteger, nullable=False, server_default="0", default=0) + reserved_micro_usd = Column(BigInteger, nullable=False, server_default="0", default=0) + settled_prompt_tokens = Column(BigInteger, nullable=False, server_default="0", default=0) + settled_completion_tokens = Column(BigInteger, nullable=False, server_default="0", default=0) + call_count = Column(Integer, nullable=False, server_default="0", default=0) + refused_count = Column(Integer, nullable=False, server_default="0", default=0) + last_call_at = Column(DateTime(timezone=True), nullable=True) + limit_source = Column( + String, + nullable=False, + server_default=LIMIT_SOURCE_STUDY_DEFAULT, + default=LIMIT_SOURCE_STUDY_DEFAULT, + ) + exhausted_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), nullable=False) + updated_at = Column(DateTime(timezone=True), nullable=False) + + +class InferenceReservation(Base): + """Append-only ledger: one row per metered inference call. + + A row is inserted in the same transaction that adds its hold to the + balance, so a hold never exists without a ledger row and vice versa. Exactly + one resolution wins (``WHERE state = 'RESERVED'``): SETTLED (actual usage), + VOIDED (proof nothing was generated: hold released), FORFEITED (outcome + unknown: full hold charged) or EXPIRED (past ``deadline_at`` when the next + reservation for the enrollment runs: full hold charged). + ``estimated_prompt_tokens`` and ``prompt_tokens`` are both kept so the + estimator's safety factor can be audited from the ledger. + """ + + __tablename__ = "inference_reservation" + __table_args__ = ( + CheckConstraint( + "state IN ('RESERVED', 'SETTLED', 'FORFEITED', 'VOIDED', 'EXPIRED')", + name="ck_inference_reservation_state", + ), + CheckConstraint("hold_micro_usd >= 0", name="ck_inference_reservation_hold"), + CheckConstraint( + "charged_micro_usd IS NULL OR charged_micro_usd >= 0", + name="ck_inference_reservation_charged", + ), + UniqueConstraint("request_id", name="uq_inference_reservation_request_id"), + Index("idx_inference_reservation_enrollment_state", "enrollment_id", "state"), + Index( + "idx_inference_reservation_open_deadline", + "deadline_at", + postgresql_where=text("state = 'RESERVED'"), + ), + Index("idx_inference_reservation_study_reserved_at", "study_id", "reserved_at"), + {"schema": "public"}, + ) + + reservation_id = Column(UUID(as_uuid=True), primary_key=True) + enrollment_id = Column( + UUID(as_uuid=True), + ForeignKey( + "public.enrollment_inference_balance.enrollment_id", ondelete="CASCADE" + ), + nullable=False, + ) + study_id = Column(UUID(as_uuid=True), nullable=False) + connection_id = Column( + UUID(as_uuid=True), + ForeignKey("public.provider_connection.connection_id", ondelete="SET NULL"), + nullable=True, + ) + model = Column(String, nullable=False) + entry_point = Column(String, nullable=False) + request_id = Column(String, nullable=False) + research_session_id = Column(UUID(as_uuid=True), nullable=True) + state = Column(String, nullable=False) + hold_micro_usd = Column(BigInteger, nullable=False) + estimated_prompt_tokens = Column(Integer, nullable=False) + output_cap_tokens = Column(Integer, nullable=False) + charged_micro_usd = Column(BigInteger, nullable=True) + prompt_tokens = Column(Integer, nullable=True) + completion_tokens = Column(Integer, nullable=True) + cached_prompt_tokens = Column(Integer, nullable=True) + usage_source = Column(String, nullable=True) + resolution_reason = Column(String, nullable=True) + upstream_status = Column(Integer, nullable=True) + finish_reason = Column(String, nullable=True) + reserved_at = Column(DateTime(timezone=True), nullable=False) + deadline_at = Column(DateTime(timezone=True), nullable=False) + resolved_at = Column(DateTime(timezone=True), nullable=True) + + +class InferenceBudgetAdjustment(Base): + """Append-only audit of limit changes (top-ups, set-limit, apply-default, backfill). + + ``(study_id, idempotency_key)`` is unique so a retried researcher action is + replayed instead of applied twice; ``request_digest`` detects a reused key + with a different body. ``in_flight_micro_usd`` records the holds open when + a limit was lowered, which bounds any overshoot against the new limit. + """ + + __tablename__ = "inference_budget_adjustment" + __table_args__ = ( + CheckConstraint( + "kind IN ('TOP_UP', 'SET_LIMIT', 'APPLY_DEFAULT', 'BACKFILL')", + name="ck_inference_budget_adjustment_kind", + ), + UniqueConstraint( + "study_id", "idempotency_key", name="uq_inference_budget_adjustment_idempotency" + ), + Index("idx_inference_budget_adjustment_enrollment", "enrollment_id", "occurred_at"), + Index("idx_inference_budget_adjustment_study", "study_id", "occurred_at"), + {"schema": "public"}, + ) + + adjustment_id = Column(UUID(as_uuid=True), primary_key=True) + enrollment_id = Column( + UUID(as_uuid=True), + ForeignKey( + "public.enrollment_inference_balance.enrollment_id", ondelete="CASCADE" + ), + nullable=False, + ) + study_id = Column(UUID(as_uuid=True), nullable=False) + kind = Column(String, nullable=False) + delta_micro_usd = Column(BigInteger, nullable=False) + limit_before_micro_usd = Column(BigInteger, nullable=False) + limit_after_micro_usd = Column(BigInteger, nullable=False) + in_flight_micro_usd = Column(BigInteger, nullable=False, server_default="0", default=0) + reason = Column(String, nullable=True) + actor = Column(String, nullable=True) + idempotency_key = Column(String, nullable=True) + request_digest = Column(String, nullable=True) + occurred_at = Column(DateTime(timezone=True), nullable=False) + + +# --------------------------------------------------------------------------- +# Issue 07: research session lifecycle and agent runs +# --------------------------------------------------------------------------- + + +class ResearchSessionV1(Base): + """One research session: the authoritative participant-activity boundary. + + Idle/resume timing lives in the study configuration, never in these columns. + + ``context_id`` is an opaque, client-scoped execution-context identifier (a + project/window instance). The partial unique index + ``uq_research_session_active_context`` enforces at most one non-terminal + session per ``(enrollment_id, context_id)``, so creation is idempotent for a + context while different windows/projects get distinct sessions. It is never a + filesystem path or an account identifier. + """ + + __tablename__ = "research_session" + __table_args__ = ( + Index("idx_research_session_enrollment_id", "enrollment_id"), + Index("idx_research_session_study_id", "study_id"), + Index("idx_research_session_state", "state"), + Index("idx_research_session_context_id", "context_id"), + Index( + "uq_research_session_active_context", + "enrollment_id", + "context_id", + unique=True, + postgresql_where=text("state NOT IN ('ended', 'revoked')"), + ), + {"schema": "public"}, + ) + + session_id = Column(UUID(as_uuid=True), primary_key=True) + enrollment_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_enrollment.enrollment_id", ondelete="CASCADE"), + nullable=False, + ) + study_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study.study_id", ondelete="CASCADE"), + nullable=False, + ) + # Opaque execution-context id (project/window instance), never a path. + context_id = Column(String, nullable=False) + state = Column(String, nullable=False) + opened_at = Column(DateTime(timezone=True), nullable=True) + # Liveness marker from heartbeats; never used to refresh activity/expiry. + last_heartbeat_at = Column(DateTime(timezone=True), nullable=True) + last_activity_at = Column(DateTime(timezone=True), nullable=True) + closed_at = Column(DateTime(timezone=True), nullable=True) + close_reason = Column(String, nullable=True) + resume_generation = Column(Integer, nullable=False, server_default="0", default=0) + manifest_digest = Column(String, nullable=False) + environment_json = Column(JSONB, nullable=False, default=dict) + # Append-only transition log (one serialized transition per entry). It lives + # on the session row because every transition belongs to exactly one session. + transitions_json = Column(JSONB, nullable=False, default=list) + created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.now) + + +class ResearchAgentRun(Base): + """One agent process (or one qualification run) against a release. + + A capability snapshot belongs to the run that captured it, so the snapshot + JSON and its capture time live on this row. A qualification snapshot may be + uploaded before any participant session exists, so ``research_session_id`` + is nullable: a run with no session is a detached qualification run. + """ + + __tablename__ = "research_agent_run" + __table_args__ = ( + Index("idx_research_agent_run_session_id", "research_session_id"), + {"schema": "public"}, + ) + + agent_run_id = Column(UUID(as_uuid=True), primary_key=True) + research_session_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_session.session_id", ondelete="CASCADE"), + nullable=True, + ) + agent_release_id = Column(String, nullable=True) + # Immutable assignment/profile identity for participant runs. Nullable for + # detached qualification runs and legacy rows. + assignment_id = Column(UUID(as_uuid=True), nullable=True) + agent_profile_id = Column(UUID(as_uuid=True), nullable=True) + profile_digest = Column(String, nullable=True) + profile_snapshot_json = Column(JSONB, nullable=True) + started_at = Column(DateTime(timezone=True), nullable=False) + ended_at = Column(DateTime(timezone=True), nullable=True) + outcome = Column(String, nullable=True) + # The full ``CapabilitySnapshotV1`` JSON (declared-vs-observed) captured by + # this run, and when it was captured (nullable: not every run snapshots). + snapshot_json = Column(JSONB, nullable=True) + snapshot_captured_at = Column(DateTime(timezone=True), nullable=True) + + +# --------------------------------------------------------------------------- +# Issue 09: idempotent telemetry ingestion and research persistence +# --------------------------------------------------------------------------- + + +class ResearchEvent(Base): + """One immutable accepted canonical telemetry fact. + + ``event_id`` is globally unique and, with the content digest, is the only + identity. ``(research_session_id, emitter_id, emitter_sequence)`` is indexed + for ordering and gap diagnostics but is deliberately NOT unique: an emitter + whose counter restarts (an IDE restart inside a live session, a re-launched + proxy) must never have its later facts deleted as "conflicts"; overlaps are + surfaced as ``EVENT_SEQUENCE_GAP`` coverage diagnostics instead. Accepted + payload bytes and provenance are never updated in place. + """ + + __tablename__ = "research_event" + __table_args__ = ( + Index("idx_research_event_enrollment_id", "enrollment_id"), + Index("idx_research_event_session_id", "research_session_id"), + Index("idx_research_event_emitter_id", "emitter_id"), + Index("idx_research_event_occurred_at", "occurred_at"), + Index("idx_research_event_event_type", "event_type"), + Index("idx_research_event_digest", "digest"), + Index("idx_research_event_study_id", "study_id"), + Index("idx_research_event_agent_run_id", "agent_run_id"), + Index("idx_research_event_retention_state", "retention_state"), + Index( + "idx_research_event_session_emitter_sequence", + "research_session_id", + "emitter_id", + "emitter_sequence", + ), + {"schema": "public"}, + ) + + event_id = Column(UUID(as_uuid=True), primary_key=True) + schema_version = Column(String, nullable=False) + event_type = Column(String, nullable=False) + source = Column(String, nullable=False) + # SET NULL: telemetry is retained/deleted by the enrollment's retention + # policy, so a removed session/enrollment row must not cascade into the + # events. `agent_run_id` stays a plain string: it is an external run + # identifier, not `research_agent_run.agent_run_id` (a UUID). + study_id = Column( + UUID(as_uuid=True), + ForeignKey("public.study.study_id", ondelete="SET NULL"), + nullable=True, + ) + enrollment_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_enrollment.enrollment_id", ondelete="SET NULL"), + nullable=True, + ) + research_session_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_session.session_id", ondelete="SET NULL"), + nullable=True, + ) + agent_run_id = Column(String, nullable=True) + emitter_id = Column(String, nullable=False) + emitter_sequence = Column(Integer, nullable=False) + occurred_at = Column(DateTime(timezone=True), nullable=False) + # The complete validated CanonicalEventV1 envelope (the event authority). + # The scalar columns are searchable projections derived from it at insert + # time (identity, ordering and joins); content lives only in the envelope. + envelope_json = Column(JSONB, nullable=False, default=dict) + digest = Column(String, nullable=False) + accepted_at = Column(DateTime(timezone=True), nullable=False) + retention_state = Column(String, nullable=False, server_default="RETAINED") + # When a retention action anonymized or tombstoned this row (additive). + anonymized_at = Column(DateTime(timezone=True), nullable=True) + + +class TelemetryBatchReceipt(Base): + """An immutable, retry-safe batch acknowledgement record.""" + + __tablename__ = "telemetry_batch_receipt" + __table_args__ = ( + Index("idx_telemetry_batch_receipt_session_id", "research_session_id"), + {"schema": "public"}, + ) + + receipt_id = Column(UUID(as_uuid=True), primary_key=True) + batch_id = Column(String, nullable=False, unique=True) + enrollment_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_enrollment.enrollment_id", ondelete="SET NULL"), + nullable=True, + ) + research_session_id = Column( + UUID(as_uuid=True), + ForeignKey("public.research_session.session_id", ondelete="SET NULL"), + nullable=True, + ) + accepted_at = Column(DateTime(timezone=True), nullable=False) + receipt_json = Column(JSONB, nullable=False) + + +# --------------------------------------------------------------------------- +# Issue 12: researcher control plane, coverage and export +# +# Researcher enablement is a single ``user.can_research`` flag (P3); there is no +# per-study ``research_researcher_role`` table. +# --------------------------------------------------------------------------- + + +#: ``kind`` values for :class:`ResearchRecord`. +RECORD_KIND_KILL_SWITCH = "KILL_SWITCH" +RECORD_KIND_HEALTH = "HEALTH" +RECORD_KIND_RELEASE_EVIDENCE = "RELEASE_EVIDENCE" +RECORD_KIND_PILOT_RUN = "PILOT_RUN" +RECORD_KIND_STUDY_PUBLICATION = "STUDY_PUBLICATION" +RECORD_KIND_STUDY_LIFECYCLE = "STUDY_LIFECYCLE" +RECORD_KIND_RETENTION_EVIDENCE = "RETENTION_EVIDENCE" + + +class ResearchRecord(Base): + """One generic append-only research record. + + Consolidates every cross-cutting evidence record into a single table: + kill-switch engagements, operational health, release evidence, pilot runs, + study publication/lifecycle audits, export audits and retention evidence + (deletion ledger + deletion-drill verification). ``kind`` selects the + family, ``scope_type``/``scope_id`` name the scoped subject (study, + revision, enrollment, export) and the full original record lives in + ``payload_json``. ``study_id``/``actor`` are nullable because export and + retention evidence is not always study- or actor-scoped. + """ + + __tablename__ = "research_record" + __table_args__ = ( + Index("idx_research_record_kind", "kind"), + Index("idx_research_record_scope", "scope_type", "scope_id"), + Index("idx_research_record_study_id", "study_id"), + Index("idx_research_record_occurred_at", "occurred_at"), + {"schema": "public"}, + ) + + record_id = Column(UUID(as_uuid=True), primary_key=True) + kind = Column(String, nullable=False) + scope_type = Column(String, nullable=True) + scope_id = Column(UUID(as_uuid=True), nullable=True) + study_id = Column(UUID(as_uuid=True), nullable=True) + actor = Column(String, nullable=True) + occurred_at = Column(DateTime(timezone=True), nullable=False) + payload_json = Column(JSONB, nullable=False, default=dict) + + +# --------------------------------------------------------------------------- +# Issue 13: pilot, operations, retention and the release gate +# --------------------------------------------------------------------------- + + +class ResearchRetentionJob(Base): + """One idempotent retention-execution job over an enrollment's stored data. + + The unique ``(enrollment_id, action)`` constraint makes withdrawal enqueue + idempotent: re-withdrawing never creates a second job. ``enrollment_id`` is + intentionally not a foreign key so the evidence survives even if an + enrollment row is later removed. + """ + + __tablename__ = "research_retention_job" + __table_args__ = ( + Index("idx_research_retention_job_enrollment", "enrollment_id"), + Index("idx_research_retention_job_state", "state"), + UniqueConstraint( + "enrollment_id", + "action", + name="uq_research_retention_job_enrollment_action", + ), + {"schema": "public"}, + ) + + job_id = Column(UUID(as_uuid=True), primary_key=True) + enrollment_id = Column(UUID(as_uuid=True), nullable=False) + action = Column(String, nullable=False) + state = Column(String, nullable=False) + attempts = Column(Integer, nullable=False, server_default="0", default=0) + created_at = Column(DateTime(timezone=True), nullable=False) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime(timezone=True), nullable=True) + last_error = Column(String, nullable=True) + evidence_digest = Column(String, nullable=True) + affected_events = Column(Integer, nullable=False, server_default="0", default=0) + # Retained ledger/verification evidence (and its digest) for this job. It + # lives here so the evidence survives the enrollment row it describes. + evidence_json = Column(JSONB, nullable=False, default=dict) + + +# --------------------------------------------------------------------------- +# Issue 11: runtime packaging and agent conformance +# +# Packaging evidence is not a table: a release's ``RuntimeManifestV2`` lives in +# ``agent_release.release_json.package_json``. Qualification is the release's +# platform test results; a terminal operator disable overrides them. +# --------------------------------------------------------------------------- + + diff --git a/src/main.py b/src/main.py index d81af84..2eaf274 100644 --- a/src/main.py +++ b/src/main.py @@ -16,26 +16,36 @@ Version: 1.0.0 """ +import ipaddress import logging +import os import threading import time from contextlib import asynccontextmanager -from typing import AsyncGenerator, Dict +from typing import AsyncGenerator, Dict, Optional -import uvicorn from dotenv import load_dotenv -from fastapi import FastAPI, Request -from fastapi.middleware.cors import CORSMiddleware -from starlette.middleware.base import BaseHTTPMiddleware -from starlette.responses import Response -from App import App -from backend.Responses import JsonResponseWithStatus, TooManyRequests -from backend.routers import router -from Code4meV2Config import Code4meV2Config +# Load the deployment `.env` BEFORE any router is imported: several modules +# (the research bootstrap signer, provider settings) read their environment at +# import time, and an operator-supplied variable must never be shadowed by a +# late `.env` load. Explicit environment variables win over the file. +load_dotenv() + +import uvicorn # noqa: E402 +from fastapi import FastAPI, Request # noqa: E402 +from fastapi.middleware.cors import CORSMiddleware # noqa: E402 +from starlette.middleware.base import BaseHTTPMiddleware # noqa: E402 +from starlette.responses import Response # noqa: E402 + +from App import App # noqa: E402 +from backend.classic_models_gate import ClassicModelsGate # noqa: E402 +from backend.Responses import JsonResponseWithStatus, TooManyRequests # noqa: E402 +from backend.routers import router # noqa: E402 +from backend.routers.research.bootstrap import signing_secret # noqa: E402 +from Code4meV2Config import Code4meV2Config # noqa: E402 # Global configuration -load_dotenv() config = Code4meV2Config() @@ -76,6 +86,103 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: _app.cleanup() +# Relay paths metered per enrollment (research budgets). See +# SimpleRateLimiter._get_rate_limit for why they get a higher per-IP floor. +METERED_INFERENCE_PATHS = frozenset( + { + "/api/research/inference/v1/chat/completions", + "/api/agent/inference", + "/api/acp/inference", + "/api/acp/chat/completions", + } +) +METERED_INFERENCE_DEFAULT_RATE_PER_HOUR = 20000 + +# Research-plane paths every enrolled participant calls continuously (session +# create/heartbeat/activity, telemetry uploads, bootstrap/capability refresh). +# They are authenticated by a signed capability or the account session and are +# bounded by the study's own heartbeat cadence, so they get the same high per-IP +# floor as the metered inference paths: a lab or office behind one NAT must not +# exhaust the default hourly quota with a handful of participants. +RESEARCH_PARTICIPANT_PATH_PREFIXES = ( + "/api/research/sessions", + "/api/research/telemetry/batches", + "/api/research/bootstrap", +) + + +def _parse_trusted_proxies(raw: Optional[str]) -> tuple[bool, frozenset[str]]: + """Parse ``FORWARDED_ALLOW_IPS`` (uvicorn's variable, same semantics). + + Returns ``(trust_all, addresses)``. A ``*`` trusts every peer; otherwise the + comma-separated addresses/networks are the only peers whose + ``X-Forwarded-For`` header is believed. The default is loopback only. + """ + value = (raw if raw is not None else "127.0.0.1").strip() + if value == "*": + return True, frozenset() + entries = frozenset(entry.strip() for entry in value.split(",") if entry.strip()) + return False, entries + + +def _is_trusted_proxy(peer: Optional[str], trust_all: bool, trusted: frozenset[str]) -> bool: + if trust_all: + return True + if not peer: + return False + if peer in trusted: + return True + try: + address = ipaddress.ip_address(peer) + except ValueError: + return False + for entry in trusted: + if "/" not in entry: + continue + try: + if address in ipaddress.ip_network(entry, strict=False): + return True + except ValueError: + continue + return False + + +def resolve_client_address( + peer: Optional[str], + forwarded_for: Optional[str], + *, + trust_all: bool, + trusted: frozenset[str], +) -> str: + """The address a request should be rate-limited by. + + Behind a trusted reverse proxy the socket peer is always the proxy, so the + real client is taken from ``X-Forwarded-For``: walking from the right, the + first entry that is not itself a trusted proxy (the entry our own proxy + appended is the one it saw on the wire; anything left of it is + client-supplied and forgeable). An untrusted peer's header is ignored. + + With ``*`` every peer is trusted, so the only safe choice is the RIGHTMOST + hop: the one the nearest proxy appended. The leftmost hop is whatever the + client sent and would let it pick its own rate-limit key. (uvicorn's own + proxy-headers middleware takes the leftmost hop under ``*``, which is why + ``nginx.prod.conf`` forwards a single hop, ``$remote_addr``, never the + client-supplied chain.) + """ + peer_value = peer or "unknown" + if not forwarded_for or not _is_trusted_proxy(peer, trust_all, trusted): + return peer_value + hops = [hop.strip() for hop in forwarded_for.split(",") if hop.strip()] + if not hops: + return peer_value + if trust_all: + return hops[-1] + for hop in reversed(hops): + if not _is_trusted_proxy(hop, trust_all, trusted): + return hop + return hops[0] + + class SimpleRateLimiter(BaseHTTPMiddleware): """ Simple rate limiting middleware for FastAPI. @@ -88,17 +195,32 @@ class SimpleRateLimiter(BaseHTTPMiddleware): locks: Threading locks for thread-safe request counting """ - def __init__(self, app: FastAPI) -> None: + def __init__( + self, + app: FastAPI, + *, + trusted_proxies: Optional[str] = None, + start_reset_thread: bool = True, + ) -> None: """ Initialize the rate limiter middleware. Args: app: The FastAPI application instance + trusted_proxies: ``FORWARDED_ALLOW_IPS`` value; defaults to the + environment variable (uvicorn reads the same one) or loopback. + start_reset_thread: tests pass ``False`` to keep the limiter inert. """ super().__init__(app) self.request_counts: Dict[str, int] = {} self.locks: Dict[str, threading.Lock] = {} - self._start_reset_thread() + # Guards the two maps above as a whole (creation of per-key locks and + # the hourly reset). Per-key locks only serialize one client's counter. + self._registry_lock = threading.Lock() + raw = trusted_proxies if trusted_proxies is not None else os.environ.get("FORWARDED_ALLOW_IPS") + self._trust_all_proxies, self._trusted_proxies = _parse_trusted_proxies(raw) + if start_reset_thread: + self._start_reset_thread() def _start_reset_thread(self) -> None: """Start the background thread that resets request counts periodically.""" @@ -118,23 +240,52 @@ def _reset_counts_periodically(self) -> None: """ while True: time.sleep(3600) # Wait for 1 hour - with threading.Lock(): - count_before = len(self.request_counts) - self.request_counts.clear() - logging.info(f"Reset {count_before} rate limit counters") + self.reset_counts() + + def reset_counts(self) -> None: + """Clear every counter (and the per-key lock map) under the registry lock.""" + with self._registry_lock: + count_before = len(self.request_counts) + self.request_counts.clear() + # Prune the per-key lock map with the counters: it must not grow + # one lock per ip:path for the lifetime of the process. + self.locks.clear() + logging.info(f"Reset {count_before} rate limit counters") def _get_rate_limit(self, endpoint: str) -> int: """ Get the rate limit for a specific endpoint. + Metered inference paths get a higher floor unless configured + explicitly: an agent loop makes hundreds of model calls per hour and a + lab behind one NAT shares the per-IP counter, while those paths are + already protected by authentication and per-enrollment budgets. + Args: endpoint: The API endpoint path Returns: The maximum number of requests allowed per hour for this endpoint """ - return config.max_request_rate_per_hour_config.get( - endpoint, config.default_max_request_rate_per_hour + configured = config.max_request_rate_per_hour_config.get(endpoint) + if configured is not None: + return configured + if endpoint in METERED_INFERENCE_PATHS or endpoint.startswith( + RESEARCH_PARTICIPANT_PATH_PREFIXES + ): + return max( + config.default_max_request_rate_per_hour, + METERED_INFERENCE_DEFAULT_RATE_PER_HOUR, + ) + return config.default_max_request_rate_per_hour + + def client_address(self, request: Request) -> str: + """The rate-limited client address (real client behind a trusted proxy).""" + return resolve_client_address( + request.client.host if request.client else None, + request.headers.get("x-forwarded-for"), + trust_all=self._trust_all_proxies, + trusted=self._trusted_proxies, ) def _get_client_key(self, request: Request) -> str: @@ -147,7 +298,7 @@ def _get_client_key(self, request: Request) -> str: Returns: A unique string combining client IP and endpoint path """ - ip = request.client.host if request.client else "unknown" + ip = self.client_address(request) endpoint = request.url.path return f"{ip}:{endpoint}" @@ -166,15 +317,18 @@ async def dispatch(self, request: Request, call_next) -> Response: endpoint = request.url.path # Ensure thread-safe access to request counts - if client_key not in self.locks: - self.locks[client_key] = threading.Lock() + with self._registry_lock: + key_lock = self.locks.setdefault(client_key, threading.Lock()) - with self.locks[client_key]: + with key_lock: current_count = self.request_counts.get(client_key, 0) rate_limit = self._get_rate_limit(endpoint) - logging.info( - f"Request sent to {endpoint} from {request.client.host if request.client else 'unknown'}." - f"\nRate limit: {current_count}/{rate_limit}\tCookies: {request.cookies}\tBody: {await request.body()}" + # Never log request cookies or bodies: they carry auth cookies, + # session capabilities and canonical telemetry content. DEBUG: a + # line per request would otherwise fill the disk over a study. + logging.debug( + f"Request sent to {endpoint} from {client_key.rsplit(':', 1)[0]}. " + f"Rate limit: {current_count}/{rate_limit}" ) if current_count >= rate_limit: logging.warning( @@ -219,6 +373,12 @@ def create_app() -> FastAPI: redoc_url="/redoc", ) + # Refuse the classic model endpoints when they are switched off (e.g. the no-GPU dev + # stack). Added first so it runs innermost, behind CORS and rate limiting. + if not config.classic_models_enabled: + app.add_middleware(ClassicModelsGate) + logging.info("Classic completion/chat models disabled (CLASSIC_MODELS_ENABLED=false)") + # Configure CORS middleware # IMPORTANT: When allow_credentials=True, Access-Control-Allow-Origin cannot be '*'. # Read allowed origins from env CORS_ALLOWED_ORIGINS (comma-separated). Fallback to common origins. @@ -243,6 +403,9 @@ def create_app() -> FastAPI: allow_methods=["*"], allow_headers=["*"], allow_credentials=True, + # Lets the research UI name downloads (study export) when it is served + # from another origin than the API, as in local development. + expose_headers=["Content-Disposition"], ) # Add rate limiting middleware @@ -271,15 +434,48 @@ def main() -> None: f"{config.server_host}:{config.server_port}" ) + # Fail fast: without the bootstrap signing secret every research bootstrap + # answers 503 and every participant spools forever. Only a test process + # (TEST_MODE) may run without it. + test_mode = os.environ.get("TEST_MODE", "false").strip().lower() in {"1", "true", "yes"} + if not test_mode and not signing_secret(): + raise SystemExit( + "BOOTSTRAP_SIGNING_SECRET is not set: the research plane cannot issue or " + "verify session capabilities. Set it in the environment or .env " + "(see .env.example) and start again." + ) + + # Multiple uvicorn workers are safe because assignment, session/revocation + # and receipt authority live in PostgreSQL/Redis, never process memory. + # ``CODE4ME_WEB_WORKERS`` (or ``WEB_CONCURRENCY``) selects the count. + workers = int( + os.environ.get("CODE4ME_WEB_WORKERS") + or os.environ.get("WEB_CONCURRENCY") + or "1" + ) + # Local development only: ``CODE4ME_DEV_RELOAD=1`` restarts the process when + # Python sources change, so a long-running dev stack never serves stale + # in-memory code (the failure mode that made a fixed admin route look + # broken). Reload requires a single worker and watches only this source + # directory, not the mounted repository's data/dist/node_modules trees. + dev_reload = os.environ.get("CODE4ME_DEV_RELOAD", "").strip().lower() in { + "1", + "true", + "yes", + } + if dev_reload: + workers = 1 uvicorn.run( "main:app", host=config.server_host, port=config.server_port, log_level="info", access_log=True, + workers=workers if workers > 1 else None, + reload=dev_reload, + reload_dirs=[os.path.dirname(os.path.abspath(__file__))] if dev_reload else None, # loop="uvloop", # http="httptools", - # workers=4, ) diff --git a/src/performance_and_analysis/EvaluationScript.py b/src/performance_and_analysis/EvaluationScript.py index b9befbc..e0ef6a0 100644 --- a/src/performance_and_analysis/EvaluationScript.py +++ b/src/performance_and_analysis/EvaluationScript.py @@ -1,223 +1,223 @@ -import requests -import time -import os -import statistics -import math -from datasets import load_dataset -import sqlite3 -import json -from datetime import datetime -import pandas as pd - -# --- Configuration --- -API_ENDPOINT = "http://localhost:8008/api/completion/request" -MODEL_IDS = [1] - -# --- Authentication Tokens --- -AUTH_TOKEN = os.environ.get('AUTH_TOKEN', 'efde6a22-2d6a-4ba3-8efa-889530dd70cc') -PROJECT_TOKEN = os.environ.get('PROJECT_TOKEN', '418c70dd-867f-4935-a5e0-ba2d7ad94f5b') -SESSION_TOKEN = os.environ.get('SESSION_TOKEN', 'c39799cc-9d25-4da5-b2b3-f67d46ae09d1') - -# --- Benchmark Settings --- -DB_NAME = "benchmark_results.db" -CSV_NAME = "benchmark_results.csv" -NUM_WARMUP_RUNS = 10 -NUM_BENCHMARK_RUNS = 1 -BENCHMARK_SUBSET_SIZE = 1600 - - -# --- Helper Functions --- - -def setup_database(db_name="benchmark_results.db"): - """Creates a SQLite database and a 'results' table with separate response fields.""" - conn = sqlite3.connect(db_name) - cursor = conn.cursor() - cursor.execute('DROP TABLE IF EXISTS results') # Recreate table with new schema - cursor.execute(''' - CREATE TABLE IF NOT EXISTS results ( - timestamp TEXT, - model_ids TEXT, - prefix TEXT, - suffix TEXT, - e2e_time_ms REAL, - error TEXT, - response_message TEXT, - meta_query_id TEXT, - completion_model_id INTEGER, - completion_model_name TEXT, - completion_text TEXT, - generation_time_ms INTEGER, - confidence REAL - ) - ''') - conn.commit() - conn.close() - print(f"Database '{db_name}' and new 'results' table are ready.") - - -def load_benchmark_data(subset_size=None): - """Loads the SantaCoder FIM dataset from Hugging Face.""" - print("Downloading and preparing the santacoder-fim-task dataset (all languages)...") - try: - dataset = load_dataset("bigcode/santacoder-fim-task")['train'] - if subset_size and subset_size < len(dataset): - print(f"Using a subset of {subset_size} samples.") - return dataset.select(range(subset_size)) - print(f"Using the full dataset with {len(dataset)} samples.") - return dataset - except Exception as e: - print(f"Error loading dataset: {e}") - return None - - -def prepare_request_payload(benchmark_item, model_ids): - """Formats a dataset item into the required JSON payload.""" - return { - "model_ids": model_ids, - "context": { - "prefix": benchmark_item['prompt'], - "suffix": benchmark_item['suffix'], - "file_name": "main.py", - "context_files": [] - }, - "store_context": True, - "contextual_telemetry": {"language_id": 45, "trigger_type_id": 1, "version_id": 1, - "document_char_length": len(benchmark_item['prompt']) + len(benchmark_item['suffix'])}, - "store_contextual_telemetry": True, - "behavioral_telemetry": {"time_since_last_completion": 5000, "typing_speed": 300, - "relative_document_position": 0.5}, - "store_behavioral_telemetry": True - } - - -def send_request_and_measure_time(session, payload, cookies): - """Sends a request and returns a dictionary with time, response, and error.""" - start_time = time.perf_counter() - response_json, error_msg = None, None - try: - response = session.post(API_ENDPOINT, json=payload, cookies=cookies, timeout=30) - response.raise_for_status() - response_json = response.json() - except requests.exceptions.RequestException as e: - error_msg = str(e) - print(f"Request failed: {error_msg}") - finally: - end_time = time.perf_counter() - - duration_ms = (end_time - start_time) * 1000 - - return { - "e2e_time_ms": duration_ms, - "response_json": response_json, - "error": error_msg - } - - -def analyze_results(timings): - """Calculates and prints key statistics from the collected timings.""" - if not timings: - print("No successful requests to analyze.") - return - n = len(timings) - mean_val = statistics.mean(timings) - median_val = statistics.median(timings) - stdev_val = statistics.stdev(timings) if n > 1 else 0 - p95 = sorted(timings)[int(n * 0.95)] if n > 19 else median_val - p99 = sorted(timings)[int(n * 0.99)] if n > 99 else median_val - ci_margin = 1.96 * (stdev_val / math.sqrt(n)) if n > 1 else 0 - ci_lower = mean_val - ci_margin - ci_upper = mean_val + ci_margin - - print("\n--- Performance Analysis ---") - print(f"Total Successful Requests: {n}") - print(f"Mean Response Time: {mean_val:.2f} ms") - print(f"Median Response Time (P50):{median_val:.2f} ms") - print(f"Standard Deviation: {stdev_val:.2f} ms") - print(f"95th Percentile (P95): {p95:.2f} ms") - print(f"99th Percentile (P99): {p99:.2f} ms") - print(f"95% Confidence Interval: [{ci_lower:.2f} ms, {ci_upper:.2f} ms]") - print("--------------------------\n") - - -# --- Main Execution --- -def main(): - """Main function to run the benchmark.""" - print("Starting End-to-End Response Time Benchmark...") - setup_database(DB_NAME) - - cookies = { - 'auth_token': AUTH_TOKEN, - 'project_token': PROJECT_TOKEN, - 'session_token': SESSION_TOKEN - } - - dataset = load_benchmark_data(subset_size=BENCHMARK_SUBSET_SIZE) - if not dataset: return - - with requests.Session() as session: - print(f"Performing {NUM_WARMUP_RUNS} warm-up requests...") - for i in range(NUM_WARMUP_RUNS): - payload = prepare_request_payload(dataset[i % len(dataset)], MODEL_IDS) - send_request_and_measure_time(session, payload, cookies) - - all_results_data = [] - print(f"\nStarting {NUM_BENCHMARK_RUNS} benchmark run(s) across {len(dataset)} samples each...") - for i in range(NUM_BENCHMARK_RUNS): - print(f"--- Starting Run {i + 1}/{NUM_BENCHMARK_RUNS} ---") - for j, item in enumerate(dataset): - payload = prepare_request_payload(item, MODEL_IDS) - result = send_request_and_measure_time(session, payload, cookies) - - # --- New Response Parsing Logic --- - response_data = result['response_json'] - completion_data = {} - message = None - meta_query_id = None - - if response_data: - message = response_data.get('message') - data = response_data.get('data', {}) - meta_query_id = data.get('meta_query_id') - completions = data.get('completions', []) - if completions: - completion_data = completions[0] # Get the first completion - - log_entry = { - "timestamp": datetime.now().isoformat(), - "model_ids": json.dumps(payload['model_ids']), - "prefix": payload['context']['prefix'], - "suffix": payload['context']['suffix'], - "e2e_time_ms": result['e2e_time_ms'], - "error": result['error'], - "response_message": message, - "meta_query_id": meta_query_id, - "completion_model_id": completion_data.get('model_id'), - "completion_model_name": completion_data.get('model_name'), - "completion_text": completion_data.get('completion'), - "generation_time_ms": completion_data.get('generation_time'), - "confidence": completion_data.get('confidence') - } - all_results_data.append(log_entry) - - if (j + 1) % 25 == 0: - print(f" ...completed {j + 1}/{len(dataset)} requests.") - - if not all_results_data: - print("No data collected. Exiting.") - return - - df = pd.DataFrame(all_results_data) - - with sqlite3.connect(DB_NAME) as conn: - df.to_sql('results', conn, if_exists='append', index=False) - print(f"Successfully saved {len(df)} records to '{DB_NAME}' in the 'results' table.") - - df.to_csv(CSV_NAME, index=False) - print(f"Successfully saved {len(df)} records to '{CSV_NAME}'.") - - successful_timings = df[df['error'].isnull()]['e2e_time_ms'].tolist() - analyze_results(successful_timings) - - -if __name__ == "__main__": +import requests +import time +import os +import statistics +import math +from datasets import load_dataset +import sqlite3 +import json +from datetime import datetime +import pandas as pd + +# --- Configuration --- +API_ENDPOINT = "http://localhost:8008/api/completion/request" +MODEL_IDS = [1] + +# --- Authentication Tokens --- +AUTH_TOKEN = os.environ.get('AUTH_TOKEN', 'efde6a22-2d6a-4ba3-8efa-889530dd70cc') +PROJECT_TOKEN = os.environ.get('PROJECT_TOKEN', '418c70dd-867f-4935-a5e0-ba2d7ad94f5b') +SESSION_TOKEN = os.environ.get('SESSION_TOKEN', 'c39799cc-9d25-4da5-b2b3-f67d46ae09d1') + +# --- Benchmark Settings --- +DB_NAME = "benchmark_results.db" +CSV_NAME = "benchmark_results.csv" +NUM_WARMUP_RUNS = 10 +NUM_BENCHMARK_RUNS = 1 +BENCHMARK_SUBSET_SIZE = 1600 + + +# --- Helper Functions --- + +def setup_database(db_name="benchmark_results.db"): + """Creates a SQLite database and a 'results' table with separate response fields.""" + conn = sqlite3.connect(db_name) + cursor = conn.cursor() + cursor.execute('DROP TABLE IF EXISTS results') # Recreate table with new schema + cursor.execute(''' + CREATE TABLE IF NOT EXISTS results ( + timestamp TEXT, + model_ids TEXT, + prefix TEXT, + suffix TEXT, + e2e_time_ms REAL, + error TEXT, + response_message TEXT, + meta_query_id TEXT, + completion_model_id INTEGER, + completion_model_name TEXT, + completion_text TEXT, + generation_time_ms INTEGER, + confidence REAL + ) + ''') + conn.commit() + conn.close() + print(f"Database '{db_name}' and new 'results' table are ready.") + + +def load_benchmark_data(subset_size=None): + """Loads the SantaCoder FIM dataset from Hugging Face.""" + print("Downloading and preparing the santacoder-fim-task dataset (all languages)...") + try: + dataset = load_dataset("bigcode/santacoder-fim-task")['train'] + if subset_size and subset_size < len(dataset): + print(f"Using a subset of {subset_size} samples.") + return dataset.select(range(subset_size)) + print(f"Using the full dataset with {len(dataset)} samples.") + return dataset + except Exception as e: + print(f"Error loading dataset: {e}") + return None + + +def prepare_request_payload(benchmark_item, model_ids): + """Formats a dataset item into the required JSON payload.""" + return { + "model_ids": model_ids, + "context": { + "prefix": benchmark_item['prompt'], + "suffix": benchmark_item['suffix'], + "file_name": "main.py", + "context_files": [] + }, + "store_context": True, + "contextual_telemetry": {"language_id": 45, "trigger_type_id": 1, "version_id": 1, + "document_char_length": len(benchmark_item['prompt']) + len(benchmark_item['suffix'])}, + "store_contextual_telemetry": True, + "behavioral_telemetry": {"time_since_last_completion": 5000, "typing_speed": 300, + "relative_document_position": 0.5}, + "store_behavioral_telemetry": True + } + + +def send_request_and_measure_time(session, payload, cookies): + """Sends a request and returns a dictionary with time, response, and error.""" + start_time = time.perf_counter() + response_json, error_msg = None, None + try: + response = session.post(API_ENDPOINT, json=payload, cookies=cookies, timeout=30) + response.raise_for_status() + response_json = response.json() + except requests.exceptions.RequestException as e: + error_msg = str(e) + print(f"Request failed: {error_msg}") + finally: + end_time = time.perf_counter() + + duration_ms = (end_time - start_time) * 1000 + + return { + "e2e_time_ms": duration_ms, + "response_json": response_json, + "error": error_msg + } + + +def analyze_results(timings): + """Calculates and prints key statistics from the collected timings.""" + if not timings: + print("No successful requests to analyze.") + return + n = len(timings) + mean_val = statistics.mean(timings) + median_val = statistics.median(timings) + stdev_val = statistics.stdev(timings) if n > 1 else 0 + p95 = sorted(timings)[int(n * 0.95)] if n > 19 else median_val + p99 = sorted(timings)[int(n * 0.99)] if n > 99 else median_val + ci_margin = 1.96 * (stdev_val / math.sqrt(n)) if n > 1 else 0 + ci_lower = mean_val - ci_margin + ci_upper = mean_val + ci_margin + + print("\n--- Performance Analysis ---") + print(f"Total Successful Requests: {n}") + print(f"Mean Response Time: {mean_val:.2f} ms") + print(f"Median Response Time (P50):{median_val:.2f} ms") + print(f"Standard Deviation: {stdev_val:.2f} ms") + print(f"95th Percentile (P95): {p95:.2f} ms") + print(f"99th Percentile (P99): {p99:.2f} ms") + print(f"95% Confidence Interval: [{ci_lower:.2f} ms, {ci_upper:.2f} ms]") + print("--------------------------\n") + + +# --- Main Execution --- +def main(): + """Main function to run the benchmark.""" + print("Starting End-to-End Response Time Benchmark...") + setup_database(DB_NAME) + + cookies = { + 'auth_token': AUTH_TOKEN, + 'project_token': PROJECT_TOKEN, + 'session_token': SESSION_TOKEN + } + + dataset = load_benchmark_data(subset_size=BENCHMARK_SUBSET_SIZE) + if not dataset: return + + with requests.Session() as session: + print(f"Performing {NUM_WARMUP_RUNS} warm-up requests...") + for i in range(NUM_WARMUP_RUNS): + payload = prepare_request_payload(dataset[i % len(dataset)], MODEL_IDS) + send_request_and_measure_time(session, payload, cookies) + + all_results_data = [] + print(f"\nStarting {NUM_BENCHMARK_RUNS} benchmark run(s) across {len(dataset)} samples each...") + for i in range(NUM_BENCHMARK_RUNS): + print(f"--- Starting Run {i + 1}/{NUM_BENCHMARK_RUNS} ---") + for j, item in enumerate(dataset): + payload = prepare_request_payload(item, MODEL_IDS) + result = send_request_and_measure_time(session, payload, cookies) + + # --- New Response Parsing Logic --- + response_data = result['response_json'] + completion_data = {} + message = None + meta_query_id = None + + if response_data: + message = response_data.get('message') + data = response_data.get('data', {}) + meta_query_id = data.get('meta_query_id') + completions = data.get('completions', []) + if completions: + completion_data = completions[0] # Get the first completion + + log_entry = { + "timestamp": datetime.now().isoformat(), + "model_ids": json.dumps(payload['model_ids']), + "prefix": payload['context']['prefix'], + "suffix": payload['context']['suffix'], + "e2e_time_ms": result['e2e_time_ms'], + "error": result['error'], + "response_message": message, + "meta_query_id": meta_query_id, + "completion_model_id": completion_data.get('model_id'), + "completion_model_name": completion_data.get('model_name'), + "completion_text": completion_data.get('completion'), + "generation_time_ms": completion_data.get('generation_time'), + "confidence": completion_data.get('confidence') + } + all_results_data.append(log_entry) + + if (j + 1) % 25 == 0: + print(f" ...completed {j + 1}/{len(dataset)} requests.") + + if not all_results_data: + print("No data collected. Exiting.") + return + + df = pd.DataFrame(all_results_data) + + with sqlite3.connect(DB_NAME) as conn: + df.to_sql('results', conn, if_exists='append', index=False) + print(f"Successfully saved {len(df)} records to '{DB_NAME}' in the 'results' table.") + + df.to_csv(CSV_NAME, index=False) + print(f"Successfully saved {len(df)} records to '{CSV_NAME}'.") + + successful_timings = df[df['error'].isnull()]['e2e_time_ms'].tolist() + analyze_results(successful_timings) + + +if __name__ == "__main__": main() \ No newline at end of file diff --git a/src/performance_and_analysis/EvaluationScriptChat.py b/src/performance_and_analysis/EvaluationScriptChat.py index b75532c..09d3d2c 100644 --- a/src/performance_and_analysis/EvaluationScriptChat.py +++ b/src/performance_and_analysis/EvaluationScriptChat.py @@ -1,232 +1,232 @@ -import requests -import time -import os -import statistics -import math -from datasets import load_dataset -import sqlite3 -import json -from datetime import datetime -import pandas as pd -import uuid - -# --- Configuration --- -API_ENDPOINT = "http://localhost:8008/api/chat/request" -MODEL_IDS = [3] -PROMPT_PREFIX = "Please explain the different components of the solution for the following problem: " - -# --- Authentication Tokens --- -AUTH_TOKEN = os.environ.get('AUTH_TOKEN', '242dfc32-73b8-4a33-88dc-9279dd63b349') -PROJECT_TOKEN = os.environ.get('PROJECT_TOKEN', 'de68bfe7-55fc-4f7c-83cc-5a5901d3416e') -SESSION_TOKEN = os.environ.get('SESSION_TOKEN', '896e8387-bedf-4f2b-9f2c-6057ea37833d') - -# --- Benchmark Settings --- -DB_NAME = "chat_benchmark_results.db" -CSV_NAME = "chat_benchmark_results.csv" -NUM_WARMUP_RUNS = 5 -NUM_BENCHMARK_RUNS = 1 -BENCHMARK_SUBSET_SIZE = 100 # Use a subset for faster testing - - -# --- Helper Functions --- - -def setup_database(db_name="chat_benchmark_results.db"): - """Creates a SQLite database and a 'results' table for chat benchmarks.""" - conn = sqlite3.connect(db_name) - cursor = conn.cursor() - cursor.execute('DROP TABLE IF EXISTS results') # Recreate table with new schema - cursor.execute(''' - CREATE TABLE IF NOT EXISTS results ( - timestamp TEXT, - model_ids TEXT, - user_prompt TEXT, - e2e_time_ms REAL, - error TEXT, - chat_id TEXT, - response_title TEXT, - assistant_completion TEXT, - generation_time_ms INTEGER, - confidence REAL - ) - ''') - conn.commit() - conn.close() - print(f"Database '{db_name}' and new 'results' table are ready.") - - -def load_benchmark_data(subset_size=None): - """Loads the MBPP dataset from Hugging Face.""" - print("Downloading and preparing the MBPP dataset...") - try: - dataset = load_dataset("mbpp", "full")['test'] - if subset_size and subset_size < len(dataset): - print(f"Using a subset of {subset_size} samples.") - return dataset.select(range(subset_size)) - print(f"Using the full dataset with {len(dataset)} samples.") - return dataset - except Exception as e: - print(f"Error loading dataset: {e}") - return None - - -def prepare_request_payload(benchmark_item, model_ids): - """Formats a dataset item into the required chat JSON payload.""" - user_message = f"{PROMPT_PREFIX}{benchmark_item['text']}" - return { - "model_ids": model_ids, - "chat_id": str(uuid.uuid4()), - "messages": [ - ["system", "You are a helpful python assistant"], - ["user", user_message] - ], - # CORRECTED: Added all required fields with placeholder data - "context": { - "prefix": "", - "suffix": "", - "file_name": "benchmark.py", - "selected_text": "", - "context_files": [] - }, - "contextual_telemetry": { - "version_id": 1, - "trigger_type_id": 1, - "language_id": 1, - "file_path": "string", - "caret_line": 0, - "document_char_length": len(user_message), - "relative_document_position": 1 - }, - "behavioral_telemetry": { - "time_since_last_shown": 0, - "time_since_last_accepted": 0, - "typing_speed": 0 - }, - "web_enabled": False - } - -def send_request_and_measure_time(session, payload, cookies): - """Sends a request and returns a dictionary with time, response, and error.""" - start_time = time.perf_counter() - response_json, error_msg = None, None - try: - response = session.post(API_ENDPOINT, json=payload, cookies=cookies, timeout=60) # Increased timeout for chat - response.raise_for_status() - response_json = response.json() - except requests.exceptions.RequestException as e: - error_msg = str(e) - print(f"Request failed: {error_msg}") - finally: - end_time = time.perf_counter() - - duration_ms = (end_time - start_time) * 1000 - - return { - "e2e_time_ms": duration_ms, - "response_json": response_json, - "error": error_msg - } - - -def analyze_results(timings): - """Calculates and prints key statistics from the collected timings.""" - if not timings: - print("No successful requests to analyze.") - return - # This function remains the same as before - n = len(timings) - mean_val = statistics.mean(timings) - median_val = statistics.median(timings) - stdev_val = statistics.stdev(timings) if n > 1 else 0 - p95 = sorted(timings)[int(n * 0.95)] if n > 19 else median_val - p99 = sorted(timings)[int(n * 0.99)] if n > 99 else median_val - ci_margin = 1.96 * (stdev_val / math.sqrt(n)) if n > 1 else 0 - ci_lower = mean_val - ci_margin - ci_upper = mean_val + ci_margin - - print("\n--- Performance Analysis ---") - print(f"Total Successful Requests: {n}") - print(f"Mean Response Time: {mean_val:.2f} ms") - print(f"Median Response Time (P50):{median_val:.2f} ms") - print(f"95th Percentile (P95): {p95:.2f} ms") - print("--------------------------\n") - - -# --- Main Execution --- -def main(): - """Main function to run the benchmark.""" - print("Starting Chat End-to-End Response Time Benchmark...") - setup_database(DB_NAME) - - cookies = { - 'auth_token': AUTH_TOKEN, - 'project_token': PROJECT_TOKEN, - 'session_token': SESSION_TOKEN - } - - dataset = load_benchmark_data(subset_size=BENCHMARK_SUBSET_SIZE) - if not dataset: return - - with requests.Session() as session: - print(f"Performing {NUM_WARMUP_RUNS} warm-up requests...") - for i in range(NUM_WARMUP_RUNS): - payload = prepare_request_payload(dataset[i % len(dataset)], MODEL_IDS) - send_request_and_measure_time(session, payload, cookies) - - all_results_data = [] - print(f"\nStarting {NUM_BENCHMARK_RUNS} benchmark run(s) across {len(dataset)} samples each...") - for i in range(NUM_BENCHMARK_RUNS): - print(f"--- Starting Run {i + 1}/{NUM_BENCHMARK_RUNS} ---") - for j, item in enumerate(dataset): - payload = prepare_request_payload(item, MODEL_IDS) - result = send_request_and_measure_time(session, payload, cookies) - - # --- New Response Parsing Logic for Chat --- - response_data = result['response_json'] - assistant_response = {} - chat_id, title = None, None - - if response_data: - chat_id = response_data.get('chat_id') - title = response_data.get('title') - history = response_data.get('history', []) - if history: - assistant_responses = history[-1].get('assistant_responses', []) - if assistant_responses: - assistant_response = assistant_responses[0] # Get the first assistant response - - log_entry = { - "timestamp": datetime.now().isoformat(), - "model_ids": json.dumps(payload['model_ids']), - "user_prompt": payload['messages'][-1][1], # Get the user message content - "e2e_time_ms": result['e2e_time_ms'], - "error": result['error'], - "chat_id": chat_id, - "response_title": title, - "assistant_completion": assistant_response.get('completion'), - "generation_time_ms": assistant_response.get('generation_time'), - "confidence": assistant_response.get('confidence') - } - all_results_data.append(log_entry) - - if (j + 1) % 10 == 0: - print(f" ...completed {j + 1}/{len(dataset)} requests.") - - if not all_results_data: - print("No data collected. Exiting.") - return - - df = pd.DataFrame(all_results_data) - - with sqlite3.connect(DB_NAME) as conn: - df.to_sql('results', conn, if_exists='append', index=False) - print(f"Successfully saved {len(df)} records to '{DB_NAME}' in the 'results' table.") - - df.to_csv(CSV_NAME, index=False) - print(f"Successfully saved {len(df)} records to '{CSV_NAME}'.") - - successful_timings = df[df['error'].isnull()]['e2e_time_ms'].tolist() - analyze_results(successful_timings) - - -if __name__ == "__main__": +import requests +import time +import os +import statistics +import math +from datasets import load_dataset +import sqlite3 +import json +from datetime import datetime +import pandas as pd +import uuid + +# --- Configuration --- +API_ENDPOINT = "http://localhost:8008/api/chat/request" +MODEL_IDS = [3] +PROMPT_PREFIX = "Please explain the different components of the solution for the following problem: " + +# --- Authentication Tokens --- +AUTH_TOKEN = os.environ.get('AUTH_TOKEN', '242dfc32-73b8-4a33-88dc-9279dd63b349') +PROJECT_TOKEN = os.environ.get('PROJECT_TOKEN', 'de68bfe7-55fc-4f7c-83cc-5a5901d3416e') +SESSION_TOKEN = os.environ.get('SESSION_TOKEN', '896e8387-bedf-4f2b-9f2c-6057ea37833d') + +# --- Benchmark Settings --- +DB_NAME = "chat_benchmark_results.db" +CSV_NAME = "chat_benchmark_results.csv" +NUM_WARMUP_RUNS = 5 +NUM_BENCHMARK_RUNS = 1 +BENCHMARK_SUBSET_SIZE = 100 # Use a subset for faster testing + + +# --- Helper Functions --- + +def setup_database(db_name="chat_benchmark_results.db"): + """Creates a SQLite database and a 'results' table for chat benchmarks.""" + conn = sqlite3.connect(db_name) + cursor = conn.cursor() + cursor.execute('DROP TABLE IF EXISTS results') # Recreate table with new schema + cursor.execute(''' + CREATE TABLE IF NOT EXISTS results ( + timestamp TEXT, + model_ids TEXT, + user_prompt TEXT, + e2e_time_ms REAL, + error TEXT, + chat_id TEXT, + response_title TEXT, + assistant_completion TEXT, + generation_time_ms INTEGER, + confidence REAL + ) + ''') + conn.commit() + conn.close() + print(f"Database '{db_name}' and new 'results' table are ready.") + + +def load_benchmark_data(subset_size=None): + """Loads the MBPP dataset from Hugging Face.""" + print("Downloading and preparing the MBPP dataset...") + try: + dataset = load_dataset("mbpp", "full")['test'] + if subset_size and subset_size < len(dataset): + print(f"Using a subset of {subset_size} samples.") + return dataset.select(range(subset_size)) + print(f"Using the full dataset with {len(dataset)} samples.") + return dataset + except Exception as e: + print(f"Error loading dataset: {e}") + return None + + +def prepare_request_payload(benchmark_item, model_ids): + """Formats a dataset item into the required chat JSON payload.""" + user_message = f"{PROMPT_PREFIX}{benchmark_item['text']}" + return { + "model_ids": model_ids, + "chat_id": str(uuid.uuid4()), + "messages": [ + ["system", "You are a helpful python assistant"], + ["user", user_message] + ], + # CORRECTED: Added all required fields with placeholder data + "context": { + "prefix": "", + "suffix": "", + "file_name": "benchmark.py", + "selected_text": "", + "context_files": [] + }, + "contextual_telemetry": { + "version_id": 1, + "trigger_type_id": 1, + "language_id": 1, + "file_path": "string", + "caret_line": 0, + "document_char_length": len(user_message), + "relative_document_position": 1 + }, + "behavioral_telemetry": { + "time_since_last_shown": 0, + "time_since_last_accepted": 0, + "typing_speed": 0 + }, + "web_enabled": False + } + +def send_request_and_measure_time(session, payload, cookies): + """Sends a request and returns a dictionary with time, response, and error.""" + start_time = time.perf_counter() + response_json, error_msg = None, None + try: + response = session.post(API_ENDPOINT, json=payload, cookies=cookies, timeout=60) # Increased timeout for chat + response.raise_for_status() + response_json = response.json() + except requests.exceptions.RequestException as e: + error_msg = str(e) + print(f"Request failed: {error_msg}") + finally: + end_time = time.perf_counter() + + duration_ms = (end_time - start_time) * 1000 + + return { + "e2e_time_ms": duration_ms, + "response_json": response_json, + "error": error_msg + } + + +def analyze_results(timings): + """Calculates and prints key statistics from the collected timings.""" + if not timings: + print("No successful requests to analyze.") + return + # This function remains the same as before + n = len(timings) + mean_val = statistics.mean(timings) + median_val = statistics.median(timings) + stdev_val = statistics.stdev(timings) if n > 1 else 0 + p95 = sorted(timings)[int(n * 0.95)] if n > 19 else median_val + p99 = sorted(timings)[int(n * 0.99)] if n > 99 else median_val + ci_margin = 1.96 * (stdev_val / math.sqrt(n)) if n > 1 else 0 + ci_lower = mean_val - ci_margin + ci_upper = mean_val + ci_margin + + print("\n--- Performance Analysis ---") + print(f"Total Successful Requests: {n}") + print(f"Mean Response Time: {mean_val:.2f} ms") + print(f"Median Response Time (P50):{median_val:.2f} ms") + print(f"95th Percentile (P95): {p95:.2f} ms") + print("--------------------------\n") + + +# --- Main Execution --- +def main(): + """Main function to run the benchmark.""" + print("Starting Chat End-to-End Response Time Benchmark...") + setup_database(DB_NAME) + + cookies = { + 'auth_token': AUTH_TOKEN, + 'project_token': PROJECT_TOKEN, + 'session_token': SESSION_TOKEN + } + + dataset = load_benchmark_data(subset_size=BENCHMARK_SUBSET_SIZE) + if not dataset: return + + with requests.Session() as session: + print(f"Performing {NUM_WARMUP_RUNS} warm-up requests...") + for i in range(NUM_WARMUP_RUNS): + payload = prepare_request_payload(dataset[i % len(dataset)], MODEL_IDS) + send_request_and_measure_time(session, payload, cookies) + + all_results_data = [] + print(f"\nStarting {NUM_BENCHMARK_RUNS} benchmark run(s) across {len(dataset)} samples each...") + for i in range(NUM_BENCHMARK_RUNS): + print(f"--- Starting Run {i + 1}/{NUM_BENCHMARK_RUNS} ---") + for j, item in enumerate(dataset): + payload = prepare_request_payload(item, MODEL_IDS) + result = send_request_and_measure_time(session, payload, cookies) + + # --- New Response Parsing Logic for Chat --- + response_data = result['response_json'] + assistant_response = {} + chat_id, title = None, None + + if response_data: + chat_id = response_data.get('chat_id') + title = response_data.get('title') + history = response_data.get('history', []) + if history: + assistant_responses = history[-1].get('assistant_responses', []) + if assistant_responses: + assistant_response = assistant_responses[0] # Get the first assistant response + + log_entry = { + "timestamp": datetime.now().isoformat(), + "model_ids": json.dumps(payload['model_ids']), + "user_prompt": payload['messages'][-1][1], # Get the user message content + "e2e_time_ms": result['e2e_time_ms'], + "error": result['error'], + "chat_id": chat_id, + "response_title": title, + "assistant_completion": assistant_response.get('completion'), + "generation_time_ms": assistant_response.get('generation_time'), + "confidence": assistant_response.get('confidence') + } + all_results_data.append(log_entry) + + if (j + 1) % 10 == 0: + print(f" ...completed {j + 1}/{len(dataset)} requests.") + + if not all_results_data: + print("No data collected. Exiting.") + return + + df = pd.DataFrame(all_results_data) + + with sqlite3.connect(DB_NAME) as conn: + df.to_sql('results', conn, if_exists='append', index=False) + print(f"Successfully saved {len(df)} records to '{DB_NAME}' in the 'results' table.") + + df.to_csv(CSV_NAME, index=False) + print(f"Successfully saved {len(df)} records to '{CSV_NAME}'.") + + successful_timings = df[df['error'].isnull()]['e2e_time_ms'].tolist() + analyze_results(successful_timings) + + +if __name__ == "__main__": main() \ No newline at end of file diff --git a/src/performance_and_analysis/benchmark_results.csv b/src/performance_and_analysis/benchmark_results.csv index 2ac3ee7..81a56b7 100644 --- a/src/performance_and_analysis/benchmark_results.csv +++ b/src/performance_and_analysis/benchmark_results.csv @@ -1,4 +1,4 @@ -timestamp,model_ids,prefix,suffix,e2e_time_ms,error,response_message,meta_query_id,completion_model_id,completion_model_name,completion_text,generation_time_ms,confidence +timestamp,model_ids,prefix,suffix,e2e_time_ms,error,response_message,meta_query_id,completion_model_id,completion_model_name,completion_text,generation_time_ms,confidence 2025-09-22T13:56:39.060324,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27,7 +27,7 @@ class Problem { } }",84.04280000104336,,Completions generated successfully. Ready for display in your IDE.,6c9846fd-58a5-48f2-b0e3-f93c40e555c6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:56:39.226354,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -61,7 +61,7 @@ class Problem { }"," } }",165.78080000181217,,Completions generated successfully. Ready for display in your IDE.,a3d0511a-e555-4563-acf1-1c062607adfc,1,deepseek-ai/deepseek-coder-1.3b-base," return new String(arr); -",116,1.0 +",116,1.0 2025-09-22T13:56:39.314124,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -89,7 +89,7 @@ class Problem { } }",87.12779999768827,,Completions generated successfully. Ready for display in your IDE.,50c4e0e5-1f5f-4446-a7b1-154c362686db,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:56:39.399312,[1],"def is_multiply_prime(a: int) -> bool: """"""Write a function that returns true if the given number is the multiplication of 3 prime numbers and false otherwise. @@ -114,7 +114,7 @@ class Problem { return True else:"," ",85.44360000087181,,Completions generated successfully. Ready for display in your IDE.,cf823418-b1c8-46f1-9c06-1bcf29fe6856,1,deepseek-ai/deepseek-coder-1.3b-base," - return False",61,1.0 + return False",61,1.0 2025-09-22T13:56:39.535994,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -154,7 +154,7 @@ class Problem { } }",136.19849999668077,,Completions generated successfully. Ready for display in your IDE.,d9229a9e-9def-4715-bac6-24019eab32ef,1,deepseek-ai/deepseek-coder-1.3b-base," int sum = 0; -",111,1.0 +",111,1.0 2025-09-22T13:56:39.636250,[1],"//We have an array 'arr' of N integers arr[1], arr[2], ..., arr[N].The // numbers in the array will be randomly ordered. Your task is to determine if // it is possible to get an array sorted in non-decreasing order by performing @@ -191,7 +191,7 @@ function move_one_ball(arr){ else{ return true;",},100.04470000421861,,Completions generated successfully. Ready for display in your IDE.,4c9f998b-4f1f-4d64-a5b8-d98df8b59650,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:56:39.855396,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -258,7 +258,7 @@ function numerical_letter_grade(grades){ return grades_array; }",219.43630000168923,,Completions generated successfully. Ready for display in your IDE.,953e6f2c-904c-4c32-b069-bd7557a482ef,1,deepseek-ai/deepseek-coder-1.3b-base," else if (grades[i] > 1.7){ -",191,1.0 +",191,1.0 2025-09-22T13:56:39.981516,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -306,7 +306,7 @@ class Problem { } }",125.68989999999758,,Completions generated successfully. Ready for display in your IDE.,8aca7d62-3fac-4050-875c-d72bead419b8,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",97,1.0 +",97,1.0 2025-09-22T13:56:40.328580,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -340,7 +340,7 @@ class Problem { }",346.68429999874206,,Completions generated successfully. Ready for display in your IDE.,b6a4008b-70fa-458b-bcdb-0800281c3db4,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < words.length; ++i) { String word = words[i]; -",320,1.0 +",320,1.0 2025-09-22T13:56:40.401109,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -379,7 +379,7 @@ class Problem { } return res; }",,72.13529999717139,,Completions generated successfully. Ready for display in your IDE.,33e1d8df-e2f8-4d93-8972-01cfec8444d9,1,deepseek-ai/deepseek-coder-1.3b-base," -}",47,1.0 +}",47,1.0 2025-09-22T13:56:40.595166,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -421,7 +421,7 @@ function compare_one(a, b){ } ",194.15789999766275,,Completions generated successfully. Ready for display in your IDE.,a27f73c3-6dce-41db-96c0-5bbb891d8d88,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (a < b[0]){ -",169,1.0 +",169,1.0 2025-09-22T13:56:40.663832,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -464,7 +464,7 @@ class Problem { return list.stream().mapToLong(x -> x).sum(); } }",68.723600001249,,Completions generated successfully. Ready for display in your IDE.,1e998f85-39b0-475b-8047-5feb652b761a,1,deepseek-ai/deepseek-coder-1.3b-base," -",40,1.0 +",40,1.0 2025-09-22T13:56:40.912634,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -529,7 +529,7 @@ class Problem { } }",248.43999999575317,,Completions generated successfully. Ready for display in your IDE.,6d79b105-60f9-4822-8051-a532ff8c4765,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (currentGrade > 3.7f) { -",218,1.0 +",218,1.0 2025-09-22T13:56:41.194537,[1],"def fibfib(n: int) -> int: """"""The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: fibfib(0) == 0 @@ -552,7 +552,7 @@ class Problem { a = b b = c c = a+b+c -",228,0.9409 +",228,0.9409 2025-09-22T13:56:41.341645,[1],"from typing import List def match_parens(lst: List[str]) -> str: @@ -588,7 +588,7 @@ def match_parens(lst: List[str]) -> str: return 'No' ",146.48559999477584,,Completions generated successfully. Ready for display in your IDE.,5109a05c-1e5e-449d-9b08-ab2b84b0b408,1,deepseek-ai/deepseek-coder-1.3b-base," elif c == ')': -",121,1.0 +",121,1.0 2025-09-22T13:56:41.439420,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -611,7 +611,7 @@ class Problem { public static long doubleTheDifference(ArrayList lst) { return lst.stream().filter(x -> x%2 == 1).mapToLong(Math::round).map(x -> x*x).sum();",},97.91650000261143,,Completions generated successfully. Ready for display in your IDE.,b525cc66-8b21-4d01-a80b-83d7957df4ad,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:56:41.528780,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -631,7 +631,7 @@ class Problem { } return l;",},89.21879999979865,,Completions generated successfully. Ready for display in your IDE.,6ef6f81d-e805-4144-83b9-5389ff0f6fd1,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:56:41.746076,[1],"//Return the largest prime factor of n. Assume n > 1 and is not a prime. // >>> largest_prime_factor(13195) // 29 @@ -648,7 +648,7 @@ function largest_prime_factor(n){ return result; }",216.6819000049145,,Completions generated successfully. Ready for display in your IDE.,4b1c7f9d-8b89-4082-ab7f-b3d4c9558e9e,1,deepseek-ai/deepseek-coder-1.3b-base," if (n % result == 0 && checkPrime(result)){ -",194,0.9823 +",194,0.9823 2025-09-22T13:56:42.000695,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -678,7 +678,7 @@ class Problem { } }",254.75179999921238,,Completions generated successfully. Ready for display in your IDE.,069191f0-e801-4955-8c12-4e0a9c459fe9,1,deepseek-ai/deepseek-coder-1.3b-base," for (long i = 0; i < n; i++) { -",206,1.0 +",206,1.0 2025-09-22T13:56:42.124004,[1],"def int_to_mini_roman(number: int) -> str: """""" Given a positive integer, obtain its roman numeral equivalent as a string, @@ -714,7 +714,7 @@ class Problem { ",122.95789999916451,,Completions generated successfully. Ready for display in your IDE.,2c465269-391b-4268-a22c-b7a752f2ef28,1,deepseek-ai/deepseek-coder-1.3b-base," number %= key -",98,1.0 +",98,1.0 2025-09-22T13:56:42.218937,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -754,7 +754,7 @@ class Problem { } }",94.39610000117682,,Completions generated successfully. Ready for display in your IDE.,1f70298d-82a1-474e-83fc-effcc41c9282,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:56:42.348577,[1],"from typing import List, Optional def prod_signs(arr: List[int]) -> Optional[int]: @@ -789,7 +789,7 @@ def prod_signs(arr: List[int]) -> Optional[int]: else: sign_arr.append(0) return sum(map(abs, arr)) * product(sign_arr)",129.5250999974087,,Completions generated successfully. Ready for display in your IDE.,1849287b-b1a2-4ee3-99f7-2911aab1c19c,1,deepseek-ai/deepseek-coder-1.3b-base," def product(lst): -",103,0.9896 +",103,0.9896 2025-09-22T13:56:42.444638,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -839,7 +839,7 @@ class Problem { } }",96.230100003595,,Completions generated successfully. Ready for display in your IDE.,b9b41a71-2c2b-4f8f-9b45-ef9805db9709,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:56:42.685087,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -863,7 +863,7 @@ function vowels_count(s){ } ",240.5473000035272,,Completions generated successfully. Ready for display in your IDE.,2b192ae7-4a1c-422f-94e9-7015ba78da2f,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=0; i< s.length ; ++i) { -",217,0.9208 +",217,0.9208 2025-09-22T13:56:42.865405,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -897,7 +897,7 @@ class Problem { } }",179.4445000050473,,Completions generated successfully. Ready for display in your IDE.,dc02a6be-00b2-45ae-ace8-3f62e5417624,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (c == '>') { -",155,0.9933 +",155,0.9933 2025-09-22T13:56:42.960302,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -937,7 +937,7 @@ class Problem { } return arrayList;",},94.71489999850746,,Completions generated successfully. Ready for display in your IDE.,9edc92be-944b-4923-946b-e53c35d5b293,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:56:43.232458,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -979,7 +979,7 @@ class Problem { } }",271.9797999961884,,Completions generated successfully. Ready for display in your IDE.,8bf973ef-40bc-4aaa-ab4f-218c71f68ff5,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < s3.length(); i++) { -",244,1.0 +",244,1.0 2025-09-22T13:56:43.389957,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1019,7 +1019,7 @@ class Problem { } }",156.99749999475898,,Completions generated successfully. Ready for display in your IDE.,60b1b9f9-3f8a-4d41-909f-f7e49145179d,1,deepseek-ai/deepseek-coder-1.3b-base," boolean isPrime = true; -",131,1.0 +",131,1.0 2025-09-22T13:56:43.749958,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -1060,7 +1060,7 @@ function max_fill(grid, capacity){ } else{ break; } -",335,1.0 +",335,1.0 2025-09-22T13:56:43.837792,[1],"//Write a function that takes a string and returns true if the string // length is a prime number or false otherwise // Examples @@ -1080,7 +1080,7 @@ function prime_length(string){ } else { return false;",},88.09750000364147,,Completions generated successfully. Ready for display in your IDE.,4f9eff4e-f371-45a7-b9e7-d388150e6ec5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:56:43.990629,[1],"//Input is a space-delimited string of numberals from 'zero' to 'nine'. // Valid choices are 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight' and 'nine'. // Return the string with numbers sorted from smallest to largest @@ -1105,7 +1105,7 @@ function sort_numbers(numbers){ return numString; }",152.56110000336776,,Completions generated successfully. Ready for display in your IDE.,dc399fd6-ec2d-4ab8-a818-9633740c4ae6,1,deepseek-ai/deepseek-coder-1.3b-base," ""one"": 1, -",114,1.0 +",114,1.0 2025-09-22T13:56:44.092477,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1154,7 +1154,7 @@ class Problem { } return s;",},100.99959999934072,,Completions generated successfully. Ready for display in your IDE.,498eaf2b-680f-4a69-9862-9ae59137550f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:56:44.189428,[1],"from typing import List def parse_nested_parens(paren_string: str) -> List[int]: @@ -1176,7 +1176,7 @@ def parse_nested_parens(paren_string: str) -> List[int]: elif paren == ')': current_depth -= 1 result.append(max_depth)",,96.98519999801647,,Completions generated successfully. Ready for display in your IDE.,393380e5-1edb-41b1-a0e8-f79f18d611cf,1,deepseek-ai/deepseek-coder-1.3b-base," - return result",72,0.9232 + return result",72,0.9232 2025-09-22T13:56:44.449626,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -1218,7 +1218,7 @@ function int_to_mini_roman(number){ ",260.3151000002981,,Completions generated successfully. Ready for display in your IDE.,f459d96e-0827-427c-bb2d-4a5b4567b34f,1,deepseek-ai/deepseek-coder-1.3b-base," //convert the given input to an array of keys from largest to smallest value -",234,0.8907 +",234,0.8907 2025-09-22T13:56:44.546273,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -1273,7 +1273,7 @@ function by_length(arr){ return arr3; }",96.39830000378424,,Completions generated successfully. Ready for display in your IDE.,149eefe0-5fa7-4fac-84aa-5b04ac28a30d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:56:44.741128,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -1341,7 +1341,7 @@ function do_algebra(operator, operand){ } ",194.2263000019011,,Completions generated successfully. Ready for display in your IDE.,e81dc2b1-a2bc-4143-99bb-49acaeb6c79f,1,deepseek-ai/deepseek-coder-1.3b-base," operator.splice(count, 1); -",165,1.0 +",165,1.0 2025-09-22T13:56:44.840098,[1],"//Write a function count_nums which takes an array of integers and returns // the number of elements which has a sum of digits > 0. // If a number is negative, then its first signed digit will be negative: @@ -1375,7 +1375,7 @@ function count_nums(arr){ } }",},99.40949999872828,,Completions generated successfully. Ready for display in your IDE.,d89e1267-cd4c-4a04-ada4-6675159f79da,1,deepseek-ai/deepseek-coder-1.3b-base," return count; -",73,1.0 +",73,1.0 2025-09-22T13:56:45.021452,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -1442,7 +1442,7 @@ function numerical_letter_grade(grades){ return grades_array; }",180.2936999956728,,Completions generated successfully. Ready for display in your IDE.,bfd77b3e-f522-43c9-ae2b-83ba501b351c,1,deepseek-ai/deepseek-coder-1.3b-base," grades_array.push(""E""); -",149,1.0 +",149,1.0 2025-09-22T13:56:45.303716,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -1483,7 +1483,7 @@ function valid_date(date){ if(!isLeapYear(year)){ return false; } else { -",255,0.9693 +",255,0.9693 2025-09-22T13:56:45.542590,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -1555,7 +1555,7 @@ function minPath(grid, k){ return path; }",239.0395999973407,,Completions generated successfully. Ready for display in your IDE.,c118f5b9-de89-4146-a18f-2416db951be9,1,deepseek-ai/deepseek-coder-1.3b-base," neighbours.push(grid[x + 1][y]); -",208,1.0 +",208,1.0 2025-09-22T13:56:45.604776,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -1591,7 +1591,7 @@ function max_fill(grid, capacity){ return maxFill; } ",61.79560000600759,,Completions generated successfully. Ready for display in your IDE.,b41f4acd-b5ac-4c50-9ee8-38b4a929be82,1,deepseek-ai/deepseek-coder-1.3b-base," -",36,1.0 +",36,1.0 2025-09-22T13:56:45.910099,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1639,7 +1639,7 @@ class Problem { } }",304.7714999993332,,Completions generated successfully. Ready for display in your IDE.,1e4f30ca-cef3-4d5c-a894-cd3fd26809fa,1,deepseek-ai/deepseek-coder-1.3b-base," if (Math.abs(indexPlanet1 - indexPlanet2) < 8) { -",274,0.9484 +",274,0.9484 2025-09-22T13:56:46.124406,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1688,7 +1688,7 @@ class Problem { } }",213.55380000022706,,Completions generated successfully. Ready for display in your IDE.,5787dbb1-9662-4f60-b0f2-6a9e152f0500,1,deepseek-ai/deepseek-coder-1.3b-base," if (max < hm.get(key)) { -",186,1.0 +",186,1.0 2025-09-22T13:56:46.250977,[1],"//Return 2^n modulo p (be aware of numerics). // >>> modp(3, 5) // 3 @@ -1710,7 +1710,7 @@ function modp(n, p){ ",126.74230000266107,,Completions generated successfully. Ready for display in your IDE.,809024ed-0f7b-4369-8f5f-59bfa147e192,1,deepseek-ai/deepseek-coder-1.3b-base," while n > 0 { -",104,0.8889 +",104,0.8889 2025-09-22T13:56:46.352337,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1771,7 +1771,7 @@ class Problem { } } }",101.11350000079256,,Completions generated successfully. Ready for display in your IDE.,24370b93-3bbf-4c6c-b5a6-f56641823959,1,deepseek-ai/deepseek-coder-1.3b-base," - ",70,1.0 + ",70,1.0 2025-09-22T13:56:46.427267,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -1810,7 +1810,7 @@ function find_max(words){ } } return max_unique_word;",,74.26700000360142,,Completions generated successfully. Ready for display in your IDE.,c6e85889-ef41-467a-940d-18d7cde96bda,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T13:56:46.470697,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1843,7 +1843,7 @@ class Problem { } return fib; } -}",43.263400002615526,,Completions generated successfully. Ready for display in your IDE.,2213393e-3bd4-458f-988d-733728beb17d,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 +}",43.263400002615526,,Completions generated successfully. Ready for display in your IDE.,2213393e-3bd4-458f-988d-733728beb17d,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 2025-09-22T13:56:46.658609,[1],"from typing import List def get_odd_collatz(n: int) -> List[int]: @@ -1877,7 +1877,7 @@ def get_odd_collatz(n: int) -> List[int]: ",187.7396999989287,,Completions generated successfully. Ready for display in your IDE.,f920c153-78e0-4e3f-8c54-dcfc7414c5c7,1,deepseek-ai/deepseek-coder-1.3b-base," current_num = current_num // 2 -",163,0.9487 +",163,0.9487 2025-09-22T13:56:46.723717,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1902,7 +1902,7 @@ class Problem { public static boolean sameChars(String s0, String s1) { return s0.chars().distinct().sorted().boxed().collect(Collectors.toList()).equals(s1.chars().distinct().sorted().boxed().collect(Collectors.toList())); }",,65.14099999913014,,Completions generated successfully. Ready for display in your IDE.,f138df19-15c6-4d0d-ae1f-6c393d104fed,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:56:46.862902,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -1926,7 +1926,7 @@ function vowels_count(s){ } ",138.54729999729898,,Completions generated successfully. Ready for display in your IDE.,b8a1037e-6c23-4b0a-b984-23a6a355b5d0,1,deepseek-ai/deepseek-coder-1.3b-base," count+=1; -",115,0.9924 +",115,0.9924 2025-09-22T13:56:46.969579,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -1950,7 +1950,7 @@ class Problem { } }",106.65840000001481,,Completions generated successfully. Ready for display in your IDE.,45f6acbd-cdea-4dda-8935-38380d9203dd,1,deepseek-ai/deepseek-coder-1.3b-base," return x; -",83,1.0 +",83,1.0 2025-09-22T13:56:47.667384,[1],"//The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fibfib(0) == 0 // fibfib(1) == 0 @@ -1980,7 +1980,7 @@ for (let i= 2;i= 0; i-- ){ -",269,0.9012 +",269,0.9012 2025-09-22T13:56:48.504102,[1],"//Input to this function is a string containing multiple groups of nested parentheses. Your goal is to // separate those group into separate strings and return the array of those. // Separate groups are balanced (each open brace is properly closed) and not nested within each other @@ -2097,7 +2097,7 @@ function separate_paren_groups(paren_string){ group = ''; } } - return groups;",,44.200899996212684,,Completions generated successfully. Ready for display in your IDE.,99ed0b90-bd29-4100-a4c9-83f6f30374cc,1,deepseek-ai/deepseek-coder-1.3b-base,,21,0.9317 + return groups;",,44.200899996212684,,Completions generated successfully. Ready for display in your IDE.,99ed0b90-bd29-4100-a4c9-83f6f30374cc,1,deepseek-ai/deepseek-coder-1.3b-base,,21,0.9317 2025-09-22T13:56:48.602308,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2147,7 +2147,7 @@ class Problem { } return ret;",},98.49430000031134,,Completions generated successfully. Ready for display in your IDE.,05cf2979-9bfe-4738-83e4-ccefe397eccd,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:56:48.740247,[1],"//Create a function which takes a string representing a file's name, and returns // 'Yes' if the the file's name is valid, and returns 'No' otherwise. // A file's name is considered to be valid if and only if all the following conditions @@ -2174,7 +2174,7 @@ function file_name_check(file_name){ } ",137.46569999784697,,Completions generated successfully. Ready for display in your IDE.,15b14cf6-9c11-40f6-aa35-65425e49ebc1,1,deepseek-ai/deepseek-coder-1.3b-base," // Write your code here... -",111,0.8412 +",111,0.8412 2025-09-22T13:56:48.817918,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2204,7 +2204,7 @@ class Problem { } return res; }",,76.87330000044312,,Completions generated successfully. Ready for display in your IDE.,b1086b67-ef16-4dc4-bb46-98f3ca0edffc,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T13:56:48.953648,[1],"def words_in_sentence(sentence: str) -> str: """""" You are given a string representing a sentence, @@ -2238,7 +2238,7 @@ class Problem { prime_words.append(word) return ' '.join(prime_words)",135.96839999809163,,Completions generated successfully. Ready for display in your IDE.,f91eb261-a19c-4cda-92c6-088397fbe62d,1,deepseek-ai/deepseek-coder-1.3b-base," for word in words: -",111,1.0 +",111,1.0 2025-09-22T13:56:49.179358,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -2275,7 +2275,7 @@ function max_fill(grid, capacity){ } ",225.1678999964497,,Completions generated successfully. Ready for display in your IDE.,39d8c21e-aa2b-42b1-ab61-3bf91ae1208b,1,deepseek-ai/deepseek-coder-1.3b-base," for(var j = 0; j < c; j++){ -",202,0.9875 +",202,0.9875 2025-09-22T13:56:49.297163,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2319,7 +2319,7 @@ class Problem { }"," } }",118.0813000028138,,Completions generated successfully. Ready for display in your IDE.,d06cb787-f4ae-4da4-9609-ddabd2eaf635,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",91,1.0 +",91,1.0 2025-09-22T13:56:49.364040,[1],"//Given an array of positive integers x. return a sorted array of all // elements that hasn't any even digit. // Note: Returned array should be sorted in increasing order. @@ -2338,7 +2338,7 @@ function unique_digits(x){ } return result.sort(function(a, b){return a-b})"," ",66.50459999946179,,Completions generated successfully. Ready for display in your IDE.,4b60bb17-a5ac-429f-bf4e-19181cdc8d3d,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:56:49.469131,[1],"//brackets is a string of ""<"" and "">"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""<"") @@ -2364,7 +2364,7 @@ function correct_bracketing(brackets){ } ",104.71849999885308,,Completions generated successfully. Ready for display in your IDE.,26390f0f-4b16-49e7-afcb-d25d7a5229d1,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",82,1.0 +",82,1.0 2025-09-22T13:56:49.760619,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2399,7 +2399,7 @@ class Problem { } }",291.0307999991346,,Completions generated successfully. Ready for display in your IDE.,b8311c84-f730-4cfa-b220-d7f1fff53558,1,deepseek-ai/deepseek-coder-1.3b-base," if (n % i == 0 || n % (i + 2) == 0) return false; -",267,1.0 +",267,1.0 2025-09-22T13:56:49.952525,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2428,7 +2428,7 @@ class Problem { } }",191.91030000365572,,Completions generated successfully. Ready for display in your IDE.,270a67a5-914a-454c-bc19-961bccb4f543,1,deepseek-ai/deepseek-coder-1.3b-base," if ((n & 1) == 0) { -",165,0.9784 +",165,0.9784 2025-09-22T13:56:50.233063,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -2500,7 +2500,7 @@ function minPath(grid, k){ return path; }",280.65820000483654,,Completions generated successfully. Ready for display in your IDE.,408df907-0480-4b71-b794-f55468ed9480,1,deepseek-ai/deepseek-coder-1.3b-base," for(let i = 1; i < neighbours.length; i++){ -",247,1.0 +",247,1.0 2025-09-22T13:56:50.362668,[1],"from typing import List, Tuple def sum_product(numbers: List[int]) -> Tuple[int, int]: @@ -2517,7 +2517,7 @@ def sum_product(numbers: List[int]) -> Tuple[int, int]: return (sum, product) ",128.79180000163615,,Completions generated successfully. Ready for display in your IDE.,e6b00e05-e1d0-42c2-b13f-534ebe14e0c3,1,deepseek-ai/deepseek-coder-1.3b-base," for i in numbers: -",105,1.0 +",105,1.0 2025-09-22T13:56:50.451609,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2548,7 +2548,7 @@ class Problem { } }",88.98670000053244,,Completions generated successfully. Ready for display in your IDE.,cdfe94d8-3bd7-48cf-9b58-c70bda248308,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:56:50.547331,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -2586,7 +2586,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",95.63859999616398,,Completions generated successfully. Ready for display in your IDE.,356ce3f7-32ff-45b5-8eb7-688ab087bf01,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:56:50.706707,[1],"//Given an array of positive integers x. return a sorted array of all // elements that hasn't any even digit. // Note: Returned array should be sorted in increasing order. @@ -2606,7 +2606,7 @@ function unique_digits(x){ ",159.0252000023611,,Completions generated successfully. Ready for display in your IDE.,3c1ca8eb-3327-4606-8ecc-92ced91856d3,1,deepseek-ai/deepseek-coder-1.3b-base," console.log(""result"", result) -",135,0.9736 +",135,0.9736 2025-09-22T13:56:50.824436,[1],"from typing import List, Tuple def sum_product(numbers: List[int]) -> Tuple[int, int]: @@ -2623,7 +2623,7 @@ def sum_product(numbers: List[int]) -> Tuple[int, int]: sum += i"," return (sum, product) ",117.29550000018207,,Completions generated successfully. Ready for display in your IDE.,634d9f3e-dfea-4c0d-a07e-94742a9fcc2d,1,deepseek-ai/deepseek-coder-1.3b-base," product *= i -",92,1.0 +",92,1.0 2025-09-22T13:56:50.982047,[1],"def is_multiply_prime(a: int) -> bool: """"""Write a function that returns true if the given number is the multiplication of 3 prime numbers and false otherwise. @@ -2649,7 +2649,7 @@ def sum_product(numbers: List[int]) -> Tuple[int, int]: ",158.13349999370985,,Completions generated successfully. Ready for display in your IDE.,24f3332c-4266-4164-93bf-a7aa9ba39361,1,deepseek-ai/deepseek-coder-1.3b-base," if n % d == 0: -",127,0.9916 +",127,0.9916 2025-09-22T13:56:51.111919,[1],"from typing import List def unique_digits(x: List[int]) -> List[int]: @@ -2675,7 +2675,7 @@ def unique_digits(x: List[int]) -> List[int]: return sorted(x) ",128.98670000140555,,Completions generated successfully. Ready for display in your IDE.,20016a6c-14cc-49ee-9bb9-3d0eff4c3bb5,1,deepseek-ai/deepseek-coder-1.3b-base," for i in x: -",104,1.0 +",104,1.0 2025-09-22T13:56:51.288131,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -2701,7 +2701,7 @@ function is_multiply_prime(a){ } }",176.51130000012927,,Completions generated successfully. Ready for display in your IDE.,d0777c38-9d52-4b07-bc38-84b0c6282dba,1,deepseek-ai/deepseek-coder-1.3b-base," check.push(prime[i]); -",135,1.0 +",135,1.0 2025-09-22T13:56:51.383547,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -2751,7 +2751,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",94.84629999496974,,Completions generated successfully. Ready for display in your IDE.,09b7370a-6f88-47fe-b960-5c76abb6abef,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:56:51.695781,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -2823,7 +2823,7 @@ function minPath(grid, k){ return path; }",311.713700000837,,Completions generated successfully. Ready for display in your IDE.,12e908f2-035b-4877-9478-1f8de1b6a904,1,deepseek-ai/deepseek-coder-1.3b-base," for(let i = 0; i < n; i++){ -",219,1.0 +",219,1.0 2025-09-22T13:56:51.999054,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2843,7 +2843,7 @@ class Problem { "," return (float)numbers.stream().mapToDouble(n -> Math.abs(n - mean)).average().getAsDouble(); } }",303.8187000056496,,Completions generated successfully. Ready for display in your IDE.,47f1d2f7-0928-4382-bb5f-6ec99a022610,1,deepseek-ai/deepseek-coder-1.3b-base," double mean = numbers.stream().mapToDouble(n -> n).average().getAsDouble(); -",279,0.9914 +",279,0.9914 2025-09-22T13:56:52.416763,[1],"//Imagine a road that's a perfectly straight infinitely long line. // n cars are driving left to right; simultaneously, a different set of n cars // are driving right to left. The two sets of cars start out being very far from @@ -2866,7 +2866,7 @@ function car_race_collision(n){ ",416.74769999372074,,Completions generated successfully. Ready for display in your IDE.,7afc3288-278b-4fda-949a-2b28e33d7e77,1,deepseek-ai/deepseek-coder-1.3b-base," // If both cars are moving towards eachother... if ((i != j && Math.abs((j - i)) % 2 == 0)){ -",393,0.9384 +",393,0.9384 2025-09-22T13:56:52.573224,[1],"def iscube(a: int) -> bool: """""" Write a function that takes an integer a and returns True @@ -2896,7 +2896,7 @@ function car_race_collision(n){ ",156.81169999879785,,Completions generated successfully. Ready for display in your IDE.,ee272af2-2b7c-4b94-8be3-a7d54e54aa06,1,deepseek-ai/deepseek-coder-1.3b-base," if (i**3 == a): -",134,0.9205 +",134,0.9205 2025-09-22T13:56:52.685466,[1],"//Given array of integers, return array in strange order. // Strange sorting, is when you start with the minimum value, // then maximum of the remaining integers, then minimum and so on. @@ -2923,7 +2923,7 @@ function strange_sort_list(lst){ tmp.splice(tmp.indexOf(output[i]), 1); }",},111.35720000311267,,Completions generated successfully. Ready for display in your IDE.,7df4749f-07f4-4b41-b51f-5f309cb7bcc1,1,deepseek-ai/deepseek-coder-1.3b-base," return output; -",83,1.0 +",83,1.0 2025-09-22T13:56:52.811134,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -2973,7 +2973,7 @@ class Problem { } }",125.77549999696203,,Completions generated successfully. Ready for display in your IDE.,9ee87c77-d441-4ed0-a9aa-7756723ed6bf,1,deepseek-ai/deepseek-coder-1.3b-base," case 'o': -",99,0.9913 +",99,0.9913 2025-09-22T13:56:52.995085,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3013,7 +3013,7 @@ class Problem { } }",183.61670000012964,,Completions generated successfully. Ready for display in your IDE.,ebe82e13-6407-4063-b550-ca96e4f87dce,1,deepseek-ai/deepseek-coder-1.3b-base," b.add(x.get(i)); -",159,1.0 +",159,1.0 2025-09-22T13:56:53.059168,[1],"//You have been tasked to write a function that receives // a hexadecimal number as a string and counts the number of hexadecimal // digits that are primes (prime number, or a prime, is a natural number @@ -3044,7 +3044,7 @@ function hex_key(num){ } return result; }",64.06440000137081,,Completions generated successfully. Ready for display in your IDE.,11abd296-dc7c-4731-86c0-e675dfcb2eb6,1,deepseek-ai/deepseek-coder-1.3b-base," -",39,1.0 +",39,1.0 2025-09-22T13:56:53.131429,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3089,7 +3089,7 @@ class Problem { } return ans.toString(); }",,71.7365999953472,,Completions generated successfully. Ready for display in your IDE.,3e4ef572-4fb3-4ade-ab88-9d16a4928035,1,deepseek-ai/deepseek-coder-1.3b-base," -}",47,1.0 +}",47,1.0 2025-09-22T13:56:53.424821,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3137,7 +3137,7 @@ class Problem { } }",292.9761000050348,,Completions generated successfully. Ready for display in your IDE.,2021873e-998c-41f0-be12-edbc29ec71ab,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < l_r.size(); i++) { -",264,0.9912 +",264,0.9912 2025-09-22T13:56:53.519369,[1],"//Given a string s, count the number of uppercase vowels in even indices. // For example: // >>> count_upper(""aBCdEf"") @@ -3156,7 +3156,7 @@ function count_upper(s){ }"," return result; }",94.36060000007274,,Completions generated successfully. Ready for display in your IDE.,cf0b46d5-61c1-480e-b1bb-5cb483666b57,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:56:54.215691,[1],"//Concatenate array of strings into a single string // >>> concatenate([]) // """" @@ -3172,7 +3172,7 @@ function concatenate(strings){ for(var i = 0 ;i < strings.length - 2; ++i ) { newString += strings[i]; }; - //console.log(""new String",672,0.9483 + //console.log(""new String",672,0.9483 2025-09-22T13:56:54.357060,[1],"//Input to this function is a string representing musical notes in a special ASCII format. // Your task is to parse this string and return array of integers corresponding to how many beats does each // not last. @@ -3196,7 +3196,7 @@ function parse_music(music_string){ } ",140.82329999655485,,Completions generated successfully. Ready for display in your IDE.,e35d1ef2-13f4-47bd-8763-fd49b8d2aede,1,deepseek-ai/deepseek-coder-1.3b-base," song.push(2); -",114,1.0 +",114,1.0 2025-09-22T13:56:54.452736,[1],"def valid_date(date: str) -> bool: """"""You have to write a function which validates a given date string and returns True if the date is valid otherwise False. @@ -3233,7 +3233,7 @@ function parse_music(music_string){ return False if len(str(date_list[2])) != 4: return False",,95.41989999706857,,Completions generated successfully. Ready for display in your IDE.,dbea6019-4710-4b7c-88f7-868d0a0cee9a,1,deepseek-ai/deepseek-coder-1.3b-base," - return True",69,1.0 + return True",69,1.0 2025-09-22T13:56:54.559829,[1],"def valid_date(date: str) -> bool: """"""You have to write a function which validates a given date string and returns True if the date is valid otherwise False. @@ -3270,7 +3270,7 @@ function parse_music(music_string){ return False return True",106.43789999448927,,Completions generated successfully. Ready for display in your IDE.,0b150836-c064-4ad5-a001-a96781490a03,1,deepseek-ai/deepseek-coder-1.3b-base," return False -",80,1.0 +",80,1.0 2025-09-22T13:56:54.686275,[1],"//The Brazilian factorial is defined as: // brazilian_factorial(n) = n! * (n-1)! * (n-2)! * ... * 1! // where n > 0 @@ -3290,7 +3290,7 @@ function special_factorial(n){ } ",126.38159999914933,,Completions generated successfully. Ready for display in your IDE.,04a6080d-6f2e-4569-b8a3-8ef2ba373eed,1,deepseek-ai/deepseek-coder-1.3b-base," let fact = 1; -",102,1.0 +",102,1.0 2025-09-22T13:56:54.854222,[1],"//The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fib4(0) -> 0 // fib4(1) -> 0 @@ -3318,7 +3318,7 @@ function fib4(n){ } ",168.04359999514418,,Completions generated successfully. Ready for display in your IDE.,67bf8d5f-c0a6-4b0e-a42e-4e8b736a1d00,1,deepseek-ai/deepseek-coder-1.3b-base," prev[2] = prev[3]; -",144,1.0 +",144,1.0 2025-09-22T13:56:54.936791,[1],"//For a given number n, find the largest number that divides n evenly, smaller than n // >>> largest_divisor(15) // 5 @@ -3332,7 +3332,7 @@ function largest_divisor(n){ return ld; }",82.43550000042887,,Completions generated successfully. Ready for display in your IDE.,619b3eda-f634-4296-9d27-8f2610da5b0b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:56:54.993054,[1],"//Check if in given array of numbers, are any two numbers closer to each other than // given threshold. // >>> has_close_elements([1.0, 2.0, 3.0], 0.5) @@ -3348,7 +3348,7 @@ function has_close_elements(numbers, threshold){ } return false; } -",55.56539999815868,,Completions generated successfully. Ready for display in your IDE.,e66237e2-d116-462c-9c95-fab76c67b999,1,deepseek-ai/deepseek-coder-1.3b-base, ,35,0.3293 +",55.56539999815868,,Completions generated successfully. Ready for display in your IDE.,e66237e2-d116-462c-9c95-fab76c67b999,1,deepseek-ai/deepseek-coder-1.3b-base, ,35,0.3293 2025-09-22T13:56:55.445198,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3392,7 +3392,7 @@ class Problem { } }",452.2540999969351,,Completions generated successfully. Ready for display in your IDE.,1fce02a4-6afb-4303-9cb5-bd9c81e7bbe9,1,deepseek-ai/deepseek-coder-1.3b-base," list.set(list.size() - 1, list.get(list.size() - 1) * operand.get(i + 1)); -",413,1.0 +",413,1.0 2025-09-22T13:56:55.666328,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -3459,7 +3459,7 @@ function numerical_letter_grade(grades){ return grades_array; }",220.86810000473633,,Completions generated successfully. Ready for display in your IDE.,7007e2a9-f55a-4c6d-bfae-4cf2b6b21106,1,deepseek-ai/deepseek-coder-1.3b-base," else if (grades[i] > 2.3){ -",192,1.0 +",192,1.0 2025-09-22T13:56:55.805558,[1],"//Given an integer. return an array that has the number of even and odd digits respectively. // Example: // >>> even_odd_count(-12) @@ -3480,7 +3480,7 @@ function even_odd_count(num){ }","} ",138.73489999969024,,Completions generated successfully. Ready for display in your IDE.,43fe4ed2-cd91-49ac-960d-2cb3d6b86f68,1,deepseek-ai/deepseek-coder-1.3b-base," return([evens, odds]); -",114,0.9814 +",114,0.9814 2025-09-22T13:56:55.893296,[1],"//triples_sum_to_zero takes an array of integers as an input. // it returns true if there are three distinct elements in the array that // sum to zero, and false otherwise. @@ -3508,7 +3508,7 @@ function triples_sum_to_zero(l){ } ",87.78199999505887,,Completions generated successfully. Ready for display in your IDE.,9ee949ec-25e7-4de9-8e25-253bae54a821,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,0.9871 +",63,0.9871 2025-09-22T13:56:56.012091,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -3585,7 +3585,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",118.5362999967765,,Completions generated successfully. Ready for display in your IDE.,45d4a86a-452f-4165-8fa9-fc130ae0d724,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",88,1.0 +",88,1.0 2025-09-22T13:56:56.552731,[1],"//From a supplied array of numbers (of length at least two) select and return two that are the closest to each // other and return them in order (smaller number, larger number). // >>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) @@ -3609,7 +3609,7 @@ function find_closest_elements(numbers){ console.log(""The smallest difference is "" + closest); console.log (""The two values with the smallest differences are:""); return [a,b].sort((a, b) => a-b); -",516,0.8487 +",516,0.8487 2025-09-22T13:56:56.647835,[1],"def count_upper(s: str) -> int: """""" Given a string s, count the number of uppercase vowels in even indices. @@ -3632,7 +3632,7 @@ D.1 Recursion Given a positive integer n, write a recursive function that returns the sum of the squares of the positive odd integers smaller than n. Assume n is greater than or equal to 3. """"""",95.33309999824269,,Completions generated successfully. Ready for display in your IDE.,7bf041c1-cd25-43e6-b79c-fb757afa130c,1,deepseek-ai/deepseek-coder-1.3b-base," - return count",72,1.0 + return count",72,1.0 2025-09-22T13:56:56.702299,[1],"//Create a function which returns the largest index of an element which // is not greater than or equal to the element immediately preceding it. If // no such element exists then return -1. The given array will not contain @@ -3653,7 +3653,7 @@ function can_arrange(arr){ return res; } ",53.89009999635164,,Completions generated successfully. Ready for display in your IDE.,aedf965c-8715-41fa-b8b9-4d5bd2d9ad16,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T13:56:56.813806,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -3725,7 +3725,7 @@ function minPath(grid, k){ return path; }",111.30150000099093,,Completions generated successfully. Ready for display in your IDE.,f5a97116-4aff-471b-b401-7e6fbdd172d7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:56:57.024819,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3769,7 +3769,7 @@ class Problem { } }",210.75950000522425,,Completions generated successfully. Ready for display in your IDE.,ab35df81-9f37-4f5c-b740-44cedf88653a,1,deepseek-ai/deepseek-coder-1.3b-base," while (n >= 10 || n < 0) { -",183,0.8902 +",183,0.8902 2025-09-22T13:56:57.196169,[1],"from typing import List def all_prefixes(string: str) -> List[str]: @@ -3781,7 +3781,7 @@ def all_prefixes(string: str) -> List[str]: return prefixes ",171.49399999470916,,Completions generated successfully. Ready for display in your IDE.,c94ccdc9-0305-409f-af3d-8fdad4f63079,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(string)): -",153,1.0 +",153,1.0 2025-09-22T13:56:57.233706,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3815,7 +3815,7 @@ class Problem { } return true; } -}",36.65739999996731,,Completions generated successfully. Ready for display in your IDE.,e14878e2-a5c9-407a-952d-2cf7fc95795e,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 +}",36.65739999996731,,Completions generated successfully. Ready for display in your IDE.,e14878e2-a5c9-407a-952d-2cf7fc95795e,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 2025-09-22T13:56:57.427735,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -3846,7 +3846,7 @@ class Problem { } }",193.84499999432592,,Completions generated successfully. Ready for display in your IDE.,bf6fadee-fea6-4139-842e-5943888d5374,1,deepseek-ai/deepseek-coder-1.3b-base," if (l.get(i) < a) { -",176,1.0 +",176,1.0 2025-09-22T13:56:57.524335,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -3895,7 +3895,7 @@ function check_dict_case(dict){ return true; }",97.11059999972349,,Completions generated successfully. Ready for display in your IDE.,289b5dc1-f499-440b-91b1-7b44936fad2e,1,deepseek-ai/deepseek-coder-1.3b-base," else { -",77,1.0 +",77,1.0 2025-09-22T13:56:57.707622,[1],"//Circular shift the digits of the integer x, shift the digits right by shift // and return the result as a string. // If shift > number of digits, return digits reversed. @@ -3917,7 +3917,7 @@ function circular_shift(x, shift){ } ",182.2602999964147,,Completions generated successfully. Ready for display in your IDE.,3a9fa82a-50bb-4f49-a76a-f78582e421c9,1,deepseek-ai/deepseek-coder-1.3b-base," //convert to String first var str = ''+x; -",165,0.903 +",165,0.903 2025-09-22T13:56:58.038415,[1],"//Create a function which takes a string representing a file's name, and returns // 'Yes' if the the file's name is valid, and returns 'No' otherwise. // A file's name is considered to be valid if and only if all the following conditions @@ -3944,7 +3944,7 @@ function file_name_check(file_name){ ",330.9120000049006,,Completions generated successfully. Ready for display in your IDE.,b57a36b4-9dfb-41c2-9a1c-7a687b154c30,1,deepseek-ai/deepseek-coder-1.3b-base," let str_2 = file_name.substring(file_name.lastIndexOf('.') + 1); -",307,0.9973 +",307,0.9973 2025-09-22T13:56:58.173411,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -3993,7 +3993,7 @@ function prime_fib(n){ } return fib(fibNum - 1); }",134.94730000093114,,Completions generated successfully. Ready for display in your IDE.,313b4de3-52db-47ac-869f-d7c9a10ccb98,1,deepseek-ai/deepseek-coder-1.3b-base," while(num >= 0){ -",115,0.9896 +",115,0.9896 2025-09-22T13:56:58.521680,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4045,7 +4045,7 @@ class Problem { }",347.96889999415725,,Completions generated successfully. Ready for display in your IDE.,4fc1e92c-0d51-47d7-9bb8-d859174cc4b6,1,deepseek-ai/deepseek-coder-1.3b-base," char first = name.charAt(0); if ((first >= 'a' && first <= 'z')) { -",327,0.9726 +",327,0.9726 2025-09-22T13:56:58.688072,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -4078,7 +4078,7 @@ class Problem { ",166.0156000070856,,Completions generated successfully. Ready for display in your IDE.,253a20e7-6497-4300-bc92-7e503ae7893d,1,deepseek-ai/deepseek-coder-1.3b-base," while (j < n - 1): -",146,1.0 +",146,1.0 2025-09-22T13:56:58.781324,[1],"//Given an array of numbers, return whether or not they are sorted // in ascending order. If array has more than 1 duplicate of the same // number, return false. Assume no negative numbers and only integers. @@ -4120,7 +4120,7 @@ function is_sorted(lst){ } ",93.0519000030472,,Completions generated successfully. Ready for display in your IDE.,4be7c619-0ced-47b8-a792-f2f5ad767c21,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:56:58.891396,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -4187,7 +4187,7 @@ function numerical_letter_grade(grades){ return grades_array; }",109.77019999700133,,Completions generated successfully. Ready for display in your IDE.,dd5c5800-f84c-4c77-b63e-246442e66071,1,deepseek-ai/deepseek-coder-1.3b-base," } -",77,1.0 +",77,1.0 2025-09-22T13:56:59.030841,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4265,7 +4265,7 @@ class Problem { } }",139.73879999684868,,Completions generated successfully. Ready for display in your IDE.,b98ac074-49c9-4dc8-bd9e-9f0d8e84bbfc,1,deepseek-ai/deepseek-coder-1.3b-base," res += ""c""; -",112,1.0 +",112,1.0 2025-09-22T13:56:59.135918,[1],"//Filter an input array of strings only for ones that contain given substring // >>> filter_by_substring([], ""a"") // [] @@ -4279,7 +4279,7 @@ function filter_by_substring(strings, substring){ } }",},104.86220000166213,,Completions generated successfully. Ready for display in your IDE.,9c31170b-3bf9-4838-b3bd-806a17e231c2,1,deepseek-ai/deepseek-coder-1.3b-base," return filtered; -",82,1.0 +",82,1.0 2025-09-22T13:56:59.242192,[1],"//Return array of prime factors of given integer in the order from smallest to largest. // Each of the factors should be arrayed number of times corresponding to how many times it appeares in factorization. // Input number should be equal to the product of all factors @@ -4302,7 +4302,7 @@ function factorize(n){ } ",105.59000000648666,,Completions generated successfully. Ready for display in your IDE.,27ee7cb8-ac8f-4df8-b0f3-fc8aa79c7e36,1,deepseek-ai/deepseek-coder-1.3b-base," factor++; -",83,1.0 +",83,1.0 2025-09-22T13:56:59.696592,[1],"from typing import Tuple def bf(planet1: str, planet2: str) -> Tuple[str, ...]: @@ -4333,7 +4333,7 @@ def bf(planet1: str, planet2: str) -> Tuple[str, ...]: else: planets = planets[end+1:start] return tuple(planets)",454.05709999613464,,Completions generated successfully. Ready for display in your IDE.,31903ab4-d785-402c-a396-56afc46961c8,1,deepseek-ai/deepseek-coder-1.3b-base," planets = ['Mercury', 'Venus', 'Earth', 'Mars', 'Jupiter', 'Saturn', 'Uranus', 'Neptune'] -",428,0.9785 +",428,0.9785 2025-09-22T13:57:00.408202,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4379,7 +4379,7 @@ class Problem { if ((char)(strongest[strength].toLowerCase()) < (char)(s[strength].toLowerCase())){ strongest = s; } -",685,0.9375 +",685,0.9375 2025-09-22T13:57:00.574935,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4411,7 +4411,7 @@ class Problem { } }",166.56229999352945,,Completions generated successfully. Ready for display in your IDE.,7add9e77-0259-4eab-83ab-1e1f34b36e45,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:57:00.708631,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4476,7 +4476,7 @@ class Problem { } }",133.5021000049892,,Completions generated successfully. Ready for display in your IDE.,4127a570-bb4f-4cd3-9c07-8bf3be41b671,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",103,1.0 +",103,1.0 2025-09-22T13:57:00.827052,[1],"//Find how many times a given substring can be found in the original string. Count overlaping cases. // >>> how_many_times("""", ""a"") // 0 @@ -4492,7 +4492,7 @@ function how_many_times(string, substring){ return count; }",117.61220000335015,,Completions generated successfully. Ready for display in your IDE.,138bba3e-c874-4f2a-afe9-4799fbab56bd,1,deepseek-ai/deepseek-coder-1.3b-base," var count = 0; -",94,1.0 +",94,1.0 2025-09-22T13:57:01.528226,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -4520,7 +4520,7 @@ function is_multiply_prime(a){ for(let j=0 ;j1;j++ ) { if((Math.floor(a/prime[j])) == ((a)/prime[j]) && a%prime[j]==0){ break; - }",674,0.9329 + }",674,0.9329 2025-09-22T13:57:01.589457,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -4560,7 +4560,7 @@ function int_to_mini_roman(number){ return output; } ",61.26190000213683,,Completions generated successfully. Ready for display in your IDE.,98dea33b-bd3e-41e7-871e-e74ba6c0644d,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:57:01.822881,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4593,7 +4593,7 @@ class Problem { }",232.80359999625944,,Completions generated successfully. Ready for display in your IDE.,914dd7a1-c415-41e1-999c-03a3af4e98c7,1,deepseek-ai/deepseek-coder-1.3b-base," long a1 = a; long b1 = b; -",204,0.9875 +",204,0.9875 2025-09-22T13:57:01.894792,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4633,7 +4633,7 @@ class Problem { } return memo[(int) n]; }",,72.13359999877866,,Completions generated successfully. Ready for display in your IDE.,7e843589-a5ab-4a6e-b5e5-c9bdff8567c0,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:57:02.052924,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -4691,7 +4691,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",158.03220000088913,,Completions generated successfully. Ready for display in your IDE.,6dd4d1cf-d64e-49a4-a297-ee0621c0aa13,1,deepseek-ai/deepseek-coder-1.3b-base," elif x > 0.0: -",132,1.0 +",132,1.0 2025-09-22T13:57:02.183529,[1],"from typing import Tuple def even_odd_count(num: int) -> Tuple[int, int]: @@ -4714,7 +4714,7 @@ def even_odd_count(num: int) -> Tuple[int, int]: ",130.24030000087805,,Completions generated successfully. Ready for display in your IDE.,e279cfda-c28c-4e6d-bd7b-93901ed5d592,1,deepseek-ai/deepseek-coder-1.3b-base," odd_count += 1 -",108,1.0 +",108,1.0 2025-09-22T13:57:02.352132,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4738,7 +4738,7 @@ class Problem { } }",168.10630000691162,,Completions generated successfully. Ready for display in your IDE.,1897fe45-01f1-4eb3-8396-0691e1ab39aa,1,deepseek-ai/deepseek-coder-1.3b-base," al.add(st.nextToken()); -",143,1.0 +",143,1.0 2025-09-22T13:57:02.444698,[1],"//Write a function that takes a string and returns an ordered version of it. // Ordered version of string, is a string where all words (separated by space) // are replaced by a new word where all the characters arranged in @@ -4760,7 +4760,7 @@ function anti_shuffle(s){ result.push(temp.join(""""));"," return result.join("" ""); }",92.18479999981355,,Completions generated successfully. Ready for display in your IDE.,e85c8ea4-a3f4-4fa2-943e-e5da66c39b1a,1,deepseek-ai/deepseek-coder-1.3b-base," }); -",68,1.0 +",68,1.0 2025-09-22T13:57:02.632430,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4788,7 +4788,7 @@ class Problem { } }",188.00690000352915,,Completions generated successfully. Ready for display in your IDE.,10666ff1-b09e-4881-8a8f-0c5ea435e7c5,1,deepseek-ai/deepseek-coder-1.3b-base," String[] words = sentence.trim().split("" ""); -",164,0.9911 +",164,0.9911 2025-09-22T13:57:02.860638,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4856,7 +4856,7 @@ class Problem { } }",227.8223999965121,,Completions generated successfully. Ready for display in your IDE.,d211c322-9e16-47d3-9050-640076dc3b6e,1,deepseek-ai/deepseek-coder-1.3b-base," seen_2_in_a_row = true; -",198,1.0 +",198,1.0 2025-09-22T13:57:02.987359,[1],"def correct_bracketing(brackets: str) -> bool: """""" brackets is a string of ""("" and "")"". return True if every opening bracket has a corresponding closing bracket. @@ -4880,7 +4880,7 @@ class Problem { ",126.14519999624463,,Completions generated successfully. Ready for display in your IDE.,0a8f4208-1d29-4d8a-ae2e-2d51438e5001,1,deepseek-ai/deepseek-coder-1.3b-base," if count < 0: -",105,1.0 +",105,1.0 2025-09-22T13:57:03.461507,[1],"//Given a positive integer n, you have to make a pile of n levels of stones. // The first level has n stones. // The number of stones in the next level is: @@ -4899,7 +4899,7 @@ function make_a_pile(n){ // console.log(""arr"", typeof arr); const nums = Math.pow((2*i + 1), 2); arr[i]=nums; -",451,0.9334 +",451,0.9334 2025-09-22T13:57:03.569791,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -4932,7 +4932,7 @@ function make_a_pile(n){ ",108.55050000100164,,Completions generated successfully. Ready for display in your IDE.,f859ef58-7d16-4ce1-9238-45ebbf1c2433,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1 -",84,1.0 +",84,1.0 2025-09-22T13:57:03.811959,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -4980,7 +4980,7 @@ class Problem { } }",241.22640000132378,,Completions generated successfully. Ready for display in your IDE.,ea6e0e93-9c8f-4da3-bc03-c32517084923,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList l_s = new ArrayList(); -",212,1.0 +",212,1.0 2025-09-22T13:57:03.961002,[1],"//Create a function that takes a value (string) representing a number // and returns the closest integer to it. If the number is equidistant // from two integers, round it away from zero. @@ -5010,7 +5010,7 @@ function closest_integer(value){ } ",149.10450000024866,,Completions generated successfully. Ready for display in your IDE.,abb1f146-f87b-4d12-b0fe-43ee5cca3ba2,1,deepseek-ai/deepseek-coder-1.3b-base," return Math.ceil(value); -",124,1.0 +",124,1.0 2025-09-22T13:57:04.208150,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -5041,7 +5041,7 @@ function same_chars(s0, s1){ ",246.5714999998454,,Completions generated successfully. Ready for display in your IDE.,7f60751e-0878-4b2e-b510-319a7961c08e,1,deepseek-ai/deepseek-coder-1.3b-base," if(s1.indexOf(s0[i]) === -1){ -",222,1.0 +",222,1.0 2025-09-22T13:57:04.324700,[1],"//You are given an array arr of integers and you need to return // sum of magnitudes of integers multiplied by product of all signs // of each number in the array, represented by 1, -1 or 0. @@ -5074,7 +5074,7 @@ function prod_signs(arr){ } } ",117.00179999752436,,Completions generated successfully. Ready for display in your IDE.,c003f44f-0f70-4f29-9cb6-23f3d731bcf5,1,deepseek-ai/deepseek-coder-1.3b-base," //your code here... -",93,0.8782 +",93,0.8782 2025-09-22T13:57:04.553368,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5122,7 +5122,7 @@ class Problem { } }",228.32370000105584,,Completions generated successfully. Ready for display in your IDE.,fb6c640f-a62e-4e8b-9159-a28d04faa565,1,deepseek-ai/deepseek-coder-1.3b-base," if (min > l_e.get(j)) { -",202,1.0 +",202,1.0 2025-09-22T13:57:04.886000,[1],"//Find the shortest palindrome that begins with a supplied string. // Algorithm idea is simple: // - Find the longest postfix of supplied string that is a palindrome. @@ -5148,7 +5148,7 @@ function make_palindrome(string){ ",332.54390000365674,,Completions generated successfully. Ready for display in your IDE.,0c983647-a724-4558-8405-485a78f6df69,1,deepseek-ai/deepseek-coder-1.3b-base," //find the longest postfix of supplied string that is a palindrome var i = string_length-1; -",309,0.948 +",309,0.948 2025-09-22T13:57:05.003843,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5213,7 +5213,7 @@ class Problem { } }",117.31439999857685,,Completions generated successfully. Ready for display in your IDE.,1fd51cff-1f11-4b4b-aeca-053a0b7deec3,1,deepseek-ai/deepseek-coder-1.3b-base," } -",81,1.0 +",81,1.0 2025-09-22T13:57:05.169869,[1],"from typing import List def sum_squares(lst: List[float]) -> int: @@ -5240,7 +5240,7 @@ def sum_squares(lst: List[float]) -> int: total = 0 for i in lst:", return total,166.15650000312598,,Completions generated successfully. Ready for display in your IDE.,71200c59-2777-4184-8299-7c6cc7ea06a2,1,deepseek-ai/deepseek-coder-1.3b-base," total += (i ** 2) -",135,0.9583 +",135,0.9583 2025-09-22T13:57:05.847614,[1],"//Your task is to write a function that returns true if a number x is a simple // power of n and false in other cases. // x is a simple power of n if n**int=x @@ -5270,7 +5270,7 @@ function is_simple_power(x, n){ ",677.3353999960818,,Completions generated successfully. Ready for display in your IDE.,255a412d-201e-452b-9ae0-2a62f59f5de1,1,deepseek-ai/deepseek-coder-1.3b-base," if (Math.pow(n, Math.floor(Math.log(x)/Math.log(n))) === x && Number.isInteger(Math.pow(n, Math.floor(Math.log(x)/Math.log(n))))); { -",654,0.9607 +",654,0.9607 2025-09-22T13:57:06.018631,[1],"//You'll be given a string of words, and your task is to count the number // of boredoms. A boredom is a sentence that starts with the word ""I"". // Sentences are delimited by '.', '?' or '!'. @@ -5289,7 +5289,7 @@ function is_bored(S){ } ",170.21540000132518,,Completions generated successfully. Ready for display in your IDE.,42dc7758-c2ce-4258-bd26-abf071f280ef,1,deepseek-ai/deepseek-coder-1.3b-base," if (/^i/.test(line)){ -",147,0.9829 +",147,0.9829 2025-09-22T13:57:06.468386,[1],"//Write a function that takes a string and returns an ordered version of it. // Ordered version of string, is a string where all words (separated by space) // are replaced by a new word where all the characters arranged in @@ -5313,7 +5313,7 @@ function anti_shuffle(s){ }",450.1712999990559,,Completions generated successfully. Ready for display in your IDE.,645fdf0d-c806-4245-9ed0-82360bac10d6,1,deepseek-ai/deepseek-coder-1.3b-base," if (!/^[a-zA-Z]+$/.test(s)) {return false;} // Split into array with each element being one word -",425,0.8954 +",425,0.8954 2025-09-22T13:57:06.572442,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5351,7 +5351,7 @@ class Problem { return lst2; }",},104.00600000139093,,Completions generated successfully. Ready for display in your IDE.,a7efe2fc-e2c6-4306-a571-0285529c41b2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",76,1.0 +",76,1.0 2025-09-22T13:57:06.653125,[1],"//Given an integer. return an array that has the number of even and odd digits respectively. // Example: // >>> even_odd_count(-12) @@ -5372,7 +5372,7 @@ function even_odd_count(num){ } ",79.53059999999823,,Completions generated successfully. Ready for display in your IDE.,2546b0f9-7486-40b4-9718-36a2a53cccf5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",56,1.0 +",56,1.0 2025-09-22T13:57:06.913312,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -5449,7 +5449,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",260.11100000323495,,Completions generated successfully. Ready for display in your IDE.,eaaf46af-3f99-41be-91b6-7bd71ae33b92,1,deepseek-ai/deepseek-coder-1.3b-base," elif isinstance(a, int) and isinstance(b, float): -",232,1.0 +",232,1.0 2025-09-22T13:57:07.252859,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5477,7 +5477,7 @@ class Problem { }",339.71139999630395,,Completions generated successfully. Ready for display in your IDE.,d10647e8-859b-45b6-81d7-12cafa74612c,1,deepseek-ai/deepseek-coder-1.3b-base," System.out.println((long)(bd.doubleValue())); return Math.round(bd.doubleValue()); -",315,0.9443 +",315,0.9443 2025-09-22T13:57:07.371373,[1],"def fizz_buzz(n: int) -> int: """"""Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. >>> fizz_buzz(50) @@ -5492,7 +5492,7 @@ class Problem { if i % 11 == 0 or i % 13 == 0:"," return x ",117.87190000177361,,Completions generated successfully. Ready for display in your IDE.,585b5095-cd95-4d93-a8bf-fab4b1a198db,1,deepseek-ai/deepseek-coder-1.3b-base," x += 1 -",92,1.0 +",92,1.0 2025-09-22T13:57:07.439679,[1],"//pairs_sum_to_zero takes an array of integers as an input. // it returns true if there are two distinct elements in the array that // sum to zero, and false otherwise. @@ -5519,7 +5519,7 @@ function pairs_sum_to_zero(l){ i = i + 1 } return false",,68.66770000488032,,Completions generated successfully. Ready for display in your IDE.,dcd8658f-8ffa-4866-9333-9c6ab460f0fd,1,deepseek-ai/deepseek-coder-1.3b-base," -}",45,1.0 +}",45,1.0 2025-09-22T13:57:07.589592,[1],"from typing import List def find_max(words: List[str]) -> str: @@ -5546,7 +5546,7 @@ def find_max(words: List[str]) -> str: return max_word ",149.39049999520648,,Completions generated successfully. Ready for display in your IDE.,ab72caef-7f1c-48a2-89b2-4fb35376f949,1,deepseek-ai/deepseek-coder-1.3b-base," max_num = unique_num -",125,1.0 +",125,1.0 2025-09-22T13:57:07.752972,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -5572,7 +5572,7 @@ function string_xor(a, b){ return result; }",163.73449999809964,,Completions generated successfully. Ready for display in your IDE.,87a8256d-bb6e-462e-8221-547c769f6d0e,1,deepseek-ai/deepseek-coder-1.3b-base," arrayXOR.push(1); -",133,1.0 +",133,1.0 2025-09-22T13:57:07.844227,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5595,7 +5595,7 @@ class Problem { } return answer;",},90.55989999615122,,Completions generated successfully. Ready for display in your IDE.,1d6b8604-e145-4909-b4ec-2a0845f68d96,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:08.563160,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -5680,7 +5680,7 @@ if __name__ == ""__main__"": import doctest doctest.testmod() -Trying",693,0.998 +Trying",693,0.998 2025-09-22T13:57:08.656794,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5720,7 +5720,7 @@ class Problem { } return memo[(int) n];",},93.57789999921806,,Completions generated successfully. Ready for display in your IDE.,3c490ffe-48a5-4477-aadb-3d80e65388c4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:57:08.979095,[1],"def get_closest_vowel(word: str) -> str: """"""You are given a word. Your task is to find the closest vowel that stands between two consonants from the right side of the word (case sensitive). @@ -5746,7 +5746,7 @@ class Problem { return word[i] return '' ",321.7603999946732,,Completions generated successfully. Ready for display in your IDE.,4d5f143e-e7d5-44e6-9ce8-3d222d0d848e,1,deepseek-ai/deepseek-coder-1.3b-base," if len(word) < 3 or set(word) & set('aeiouAEIOU') == {}: -",297,0.9723 +",297,0.9723 2025-09-22T13:57:09.049164,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5775,7 +5775,7 @@ class Problem { } return (sqrt * 2) % p; }",,69.8095000043395,,Completions generated successfully. Ready for display in your IDE.,4e255e07-8fa4-4f74-bb99-8e8dec40c6c1,1,deepseek-ai/deepseek-coder-1.3b-base," -}",44,1.0 +}",44,1.0 2025-09-22T13:57:09.155606,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5809,7 +5809,7 @@ class Problem { } }",106.08680000586901,,Completions generated successfully. Ready for display in your IDE.,a39fee58-0655-4687-a6f2-01496ca4e86d,1,deepseek-ai/deepseek-coder-1.3b-base," open++; -",83,1.0 +",83,1.0 2025-09-22T13:57:09.333343,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5851,7 +5851,7 @@ class Problem { } }",177.9549999992014,,Completions generated successfully. Ready for display in your IDE.,afa2d577-d665-442a-940f-374d02306f43,1,deepseek-ai/deepseek-coder-1.3b-base," if (c1 < 0) break; -",152,1.0 +",152,1.0 2025-09-22T13:57:09.623819,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5887,7 +5887,7 @@ class Problem { } }",290.3502000044682,,Completions generated successfully. Ready for display in your IDE.,2603564c-022b-41a3-a333-db002fb7300f,1,deepseek-ai/deepseek-coder-1.3b-base," for (int k = j + 1; k < arrayList.size(); k++) { -",265,1.0 +",265,1.0 2025-09-22T13:57:09.781080,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5924,7 +5924,7 @@ class Problem { } }",157.28390000003856,,Completions generated successfully. Ready for display in your IDE.,97638c4b-e5dc-4096-be30-03fa42114af7,1,deepseek-ai/deepseek-coder-1.3b-base," open_paren++; -",132,1.0 +",132,1.0 2025-09-22T13:57:09.996811,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -5977,7 +5977,7 @@ class Problem { } }",215.35340000264114,,Completions generated successfully. Ready for display in your IDE.,911decbd-3db4-4489-9522-e12fc241b608,1,deepseek-ai/deepseek-coder-1.3b-base," newArr.add((long)minValue); -",184,0.9495 +",184,0.9495 2025-09-22T13:57:10.085341,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6005,7 +6005,7 @@ class Problem { } }",87.58039999520406,,Completions generated successfully. Ready for display in your IDE.,e46dbe42-6307-4ad0-a66b-d378efef26a3,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:10.262141,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6043,7 +6043,7 @@ class Problem { } }",177.5119000012637,,Completions generated successfully. Ready for display in your IDE.,628e5e30-73a5-429a-91ba-bfcb67b446d3,1,deepseek-ai/deepseek-coder-1.3b-base," res.add((long)4l); -",151,0.967 +",151,0.967 2025-09-22T13:57:10.354675,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6069,7 +6069,7 @@ class Problem { } }",91.63819999957923,,Completions generated successfully. Ready for display in your IDE.,8fa723e3-0ce0-4bb1-b058-045d2e22db73,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:10.470137,[1],"//Given a positive integer n, return the product of the odd digits. // Return 0 if all digits are even. // For example: @@ -6102,7 +6102,7 @@ function digits(n){ ",114.98140000185231,,Completions generated successfully. Ready for display in your IDE.,ababbb5c-406b-46db-993c-3de30cef94dd,1,deepseek-ai/deepseek-coder-1.3b-base," return 0; -",93,1.0 +",93,1.0 2025-09-22T13:57:11.171840,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -6134,7 +6134,7 @@ function is_multiply_prime(a){ for(var j=0;j int: """""" Create a function that takes a value (string) representing a number @@ -6159,7 +6159,7 @@ function is_multiply_prime(a){ if abs(int(value)-float(value)) < 0.5: return integer_val else: - return integer_val+1 if ((abs(int(value)-float(value)) > 0.5) and (value >= 0)) else integer_val-",670,0.9465 + return integer_val+1 if ((abs(int(value)-float(value)) > 0.5) and (value >= 0)) else integer_val-",670,0.9465 2025-09-22T13:57:12.281348,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -6198,7 +6198,7 @@ function valid_date(date){ } ",416.6438000029302,,Completions generated successfully. Ready for display in your IDE.,c57ce04e-3082-46f0-913e-124bbd587bd0,1,deepseek-ai/deepseek-coder-1.3b-base," }else if((month == 4 || month == 6 || month == 9 || month == 11) && (day < 1 || day > 30)){ -",389,0.9906 +",389,0.9906 2025-09-22T13:57:12.408352,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6230,7 +6230,7 @@ class Problem { }"," } }",126.27779999456834,,Completions generated successfully. Ready for display in your IDE.,4e95f963-f33f-45fd-8a2d-c796cf7b3ab2,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",97,1.0 +",97,1.0 2025-09-22T13:57:12.733563,[1],"//In this Kata, you have to sort an array of non-negative integers according to // number of ones in their binary representation in ascending order. // For similar number of ones, sort based on decimal value. @@ -6251,7 +6251,7 @@ function sort_array(arr){ }); }",325.03279999946244,,Completions generated successfully. Ready for display in your IDE.,f1647dae-1b1f-47b7-98f0-d1a2b32e7d92,1,deepseek-ai/deepseek-coder-1.3b-base," let bBinary = b.toString(2).split('').filter(el => el === '1').length; -",299,1.0 +",299,1.0 2025-09-22T13:57:12.817426,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6278,7 +6278,7 @@ class Problem { } }",83.6866999961785,,Completions generated successfully. Ready for display in your IDE.,fff320bf-bf8d-407f-b894-eb60becbc345,1,deepseek-ai/deepseek-coder-1.3b-base," } -",60,1.0 +",60,1.0 2025-09-22T13:57:12.895544,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6307,7 +6307,7 @@ class Problem { long oranges = Long.parseLong(splitted[3]); return n - apples - oranges; }",,77.93690000107745,,Completions generated successfully. Ready for display in your IDE.,1a0f950b-a23f-4e87-9b50-79bf3624a1e6,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T13:57:13.021171,[1],"//You are given an array of integers. // You need to find the largest prime value and return the sum of its digits. // Examples: @@ -6338,7 +6338,7 @@ function skjkasdkd(lst){ } return sum; }",125.56960000074469,,Completions generated successfully. Ready for display in your IDE.,83a0739d-d89a-4f46-8906-fce910114f31,1,deepseek-ai/deepseek-coder-1.3b-base," function prime(n){ -",101,1.0 +",101,1.0 2025-09-22T13:57:13.125404,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6376,7 +6376,7 @@ class Problem { } }",104.14669999590842,,Completions generated successfully. Ready for display in your IDE.,1d1b06e6-b304-44a5-8587-39351b08bd11,1,deepseek-ai/deepseek-coder-1.3b-base," } -",79,1.0 +",79,1.0 2025-09-22T13:57:13.212727,[1],"//Create a function that takes a value (string) representing a number // and returns the closest integer to it. If the number is equidistant // from two integers, round it away from zero. @@ -6406,7 +6406,7 @@ function closest_integer(value){ } ",87.50240000517806,,Completions generated successfully. Ready for display in your IDE.,77faac8d-fe4c-4c45-81ce-f69443f4dc56,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:13.301742,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6434,7 +6434,7 @@ class Problem { } return p;",},88.52379999734694,,Completions generated successfully. Ready for display in your IDE.,deccfacc-d3eb-4693-ac25-8b536a5717b8,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:13.459532,[1],"from typing import List def pairs_sum_to_zero(l: List[int]) -> bool: @@ -6458,7 +6458,7 @@ def pairs_sum_to_zero(l: List[int]) -> bool: return False ",157.1637999950326,,Completions generated successfully. Ready for display in your IDE.,3f894919-0c3d-4a73-9c39-c507d9d159af,1,deepseek-ai/deepseek-coder-1.3b-base," if (x + y == 0): -",134,0.8809 +",134,0.8809 2025-09-22T13:57:13.504909,[1],"def remove_vowels(text: str) -> str: """""" remove_vowels is a function that takes string and returns string without vowels. @@ -6474,7 +6474,7 @@ def pairs_sum_to_zero(l: List[int]) -> bool: 'zbcd' """""" "," return ''.join(filter(lambda x: x not in vowels, text)) -",45.010699999693315,,Completions generated successfully. Ready for display in your IDE.,c60aa39c-0099-433b-a535-8ed0cbbd9b65,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",45.010699999693315,,Completions generated successfully. Ready for display in your IDE.,c60aa39c-0099-433b-a535-8ed0cbbd9b65,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:57:13.633225,[1],"//Given a string 'text', return its md5 hash equivalent string. // If 'text' is an empty string, return undefined. // >>> string_to_md5(""Hello world"") @@ -6485,7 +6485,7 @@ function string_to_md5(text){ } ",128.4534000005806,,Completions generated successfully. Ready for display in your IDE.,77892d9e-edaa-4872-9da4-67a8061644a1,1,deepseek-ai/deepseek-coder-1.3b-base," console.log('undefined'); -",107,0.9016 +",107,0.9016 2025-09-22T13:57:13.686653,[1],"from typing import List def find_max(words: List[str]) -> str: @@ -6511,7 +6511,7 @@ def find_max(words: List[str]) -> str: max_word = word return max_word ",53.40240000077756,,Completions generated successfully. Ready for display in your IDE.,23c5794b-2d60-4105-9741-feeb3bae61db,1,deepseek-ai/deepseek-coder-1.3b-base," -",30,1.0 +",30,1.0 2025-09-22T13:57:13.816257,[1],"def fruit_distribution(s: str, n: int) -> int: """""" In this task, you will be given a string that represents a number of apples and oranges @@ -6538,7 +6538,7 @@ def find_max(words: List[str]) -> str: ",129.4449999986682,,Completions generated successfully. Ready for display in your IDE.,f760b7f9-7eaa-4506-9c14-b53796f51a3e,1,deepseek-ai/deepseek-coder-1.3b-base," except ValueError: -",99,1.0 +",99,1.0 2025-09-22T13:57:13.934180,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -6577,7 +6577,7 @@ function valid_date(date){ } ",117.72999999811873,,Completions generated successfully. Ready for display in your IDE.,faa514c0-3951-44ba-9c9a-960ba8e65604,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",90,1.0 +",90,1.0 2025-09-22T13:57:14.369409,[1],"//Filter an input array of strings only for ones that contain given substring // >>> filter_by_substring([], ""a"") // [] @@ -6593,7 +6593,7 @@ function filter_by_substring(strings, substring){ //loop through the string and check each one against our search term for (var i = 0 ; i < strings.length; ++i) { -",412,0.9216 +",412,0.9216 2025-09-22T13:57:14.744254,[1],"//You are given 2 words. You need to return true if the second word or any of its rotations is a substring in the first word // >>> cycpattern_check(""abcd"", ""abd"") // false @@ -6615,7 +6615,7 @@ function cycpattern_check(a, b){ } ",374.3818000002648,,Completions generated successfully. Ready for display in your IDE.,94723c98-8f64-4d54-bde1-b2936f0f866b,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i = 0 ;i < Math.floor((Math.sqrt(a.length))/3)*4 + 5;) { -",351,0.8684 +",351,0.8684 2025-09-22T13:57:15.034347,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6647,7 +6647,7 @@ class Problem { return false; } }",289.7977999964496,,Completions generated successfully. Ready for display in your IDE.,b7a62009-d2af-4f7f-b48b-bc609e617ed7,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < l.size() - 1; i++) { -",265,0.9966 +",265,0.9966 2025-09-22T13:57:15.186704,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6679,7 +6679,7 @@ class Problem { } }",152.334599995811,,Completions generated successfully. Ready for display in your IDE.,62e4bd91-e966-4d1c-a79e-465e6d5522d1,1,deepseek-ai/deepseek-coder-1.3b-base," long max_depth = 0; -",126,0.9158 +",126,0.9158 2025-09-22T13:57:15.299286,[1],"def strlen(string: str) -> int: """""" Return length of given string >>> strlen('') @@ -6692,7 +6692,7 @@ class Problem { ",112.45220000273548,,Completions generated successfully. Ready for display in your IDE.,527b0710-12a6-4fef-905a-8c269429325e,1,deepseek-ai/deepseek-coder-1.3b-base," length += 1 -",91,1.0 +",91,1.0 2025-09-22T13:57:15.419617,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -6769,7 +6769,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",119.57280000206083,,Completions generated successfully. Ready for display in your IDE.,13dd0e4d-b814-4c09-bce0-c8d7356f0013,1,deepseek-ai/deepseek-coder-1.3b-base," return None -",88,1.0 +",88,1.0 2025-09-22T13:57:15.536682,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6820,7 +6820,7 @@ class Problem { } }",116.76450000231853,,Completions generated successfully. Ready for display in your IDE.,c90a2f80-f0ec-4400-a1d0-f2195194c6b7,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",89,1.0 +",89,1.0 2025-09-22T13:57:15.600916,[1],"//Task // We are given two strings s and c, you have to deleted all the characters in s that are equal to any character in c // then check if the result string is palindrome. @@ -6844,7 +6844,7 @@ function reverse_delete(s, c){ result.push(filterString, false); } return result;",,64.54600000142818,,Completions generated successfully. Ready for display in your IDE.,5583d8ef-e993-409a-b93a-a789cbdb59aa,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:57:15.694800,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -6895,7 +6895,7 @@ class Problem { } }",93.47400000115158,,Completions generated successfully. Ready for display in your IDE.,084752ae-03fd-4510-89f5-8f8964874633,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:57:15.868884,[1],"from typing import List def sum_squares(lst: List[int]) -> int: @@ -6917,7 +6917,7 @@ def sum_squares(lst: List[int]) -> int: lst[index] = lst[index] ** 2"," lst[index] = lst[index] ** 3 return sum(lst)",173.9201999953366,,Completions generated successfully. Ready for display in your IDE.,aba7c6ec-7be7-452b-9de4-4819ccb68749,1,deepseek-ai/deepseek-coder-1.3b-base," elif index % 4 != 0: -",150,0.9753 +",150,0.9753 2025-09-22T13:57:15.956244,[1],"//Write a function that accepts two arrays of strings and returns the array that has // total number of chars in the all strings of the array less than the other array. // if the two arrays have the same number of chars, return the first array. @@ -6948,7 +6948,7 @@ function total_match(lst1, lst2){ } ",87.34540000295965,,Completions generated successfully. Ready for display in your IDE.,e4b67de6-6754-4263-a43b-a32e68c53e7a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:57:16.070605,[1],"from typing import List def max_fill(grid: List[List[int]], capacity: int) -> int: @@ -6985,7 +6985,7 @@ def max_fill(grid: List[List[int]], capacity: int) -> int: if 1 in row[i:i+capacity]: times += 1",,114.2318999991403,,Completions generated successfully. Ready for display in your IDE.,b89e1fdc-a136-4d2b-82c9-0e0b2716ed2a,1,deepseek-ai/deepseek-coder-1.3b-base," - return times",90,0.9921 + return times",90,0.9921 2025-09-22T13:57:16.165397,[1],"from typing import Optional def string_to_md5(text: str) -> Optional[str]: @@ -7001,7 +7001,7 @@ def string_to_md5(text: str) -> Optional[str]: return None"," return hashlib.md5(text.encode('utf-8')).hexdigest() ",94.16659999988042,,Completions generated successfully. Ready for display in your IDE.,4e7238d9-4ce7-474b-ab97-f7d1549233e1,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",72,1.0 +",72,1.0 2025-09-22T13:57:16.239178,[1],"//Create a function that returns an array (a, b), where 'a' is // the largest of negative integers, and 'b' is the smallest // of positive integers in an array. @@ -7015,7 +7015,7 @@ def string_to_md5(text: str) -> Optional[str]: // [undefined, undefined] function largest_smallest_integers(lst){ return[lst.filter(x=>x<0).sort((a,b)=>b-a)[0],lst.filter(x=>x>0).sort((a,b)=>a-b)[0]]",,73.853699999745,,Completions generated successfully. Ready for display in your IDE.,5ec41d61-060c-4ceb-8f32-823aa638327b,1,deepseek-ai/deepseek-coder-1.3b-base,"; -}",52,1.0 +}",52,1.0 2025-09-22T13:57:16.323490,[1],"//Given an integer. return an array that has the number of even and odd digits respectively. // Example: // >>> even_odd_count(-12) @@ -7036,7 +7036,7 @@ function even_odd_count(num){ } ",83.82540000457084,,Completions generated successfully. Ready for display in your IDE.,12d74332-5615-4260-a0e4-d3acccd1acb3,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:57:16.649248,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -7075,7 +7075,7 @@ function find_max(words){ return max_unique_word; }",325.37479999882635,,Completions generated successfully. Ready for display in your IDE.,065dc21a-902f-4752-9fc9-8769d542ff55,1,deepseek-ai/deepseek-coder-1.3b-base," else if(Object.keys(unique_word).length > Object.keys(max_unique).length){ -",298,1.0 +",298,1.0 2025-09-22T13:57:16.825574,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7153,7 +7153,7 @@ class Problem { } }",176.62779999955092,,Completions generated successfully. Ready for display in your IDE.,c99968ce-534d-4f5f-9306-82331e3e6141,1,deepseek-ai/deepseek-coder-1.3b-base," if (num >= 40) { -",144,1.0 +",144,1.0 2025-09-22T13:57:17.256635,[1],"//Write a function that returns true if the object q will fly, and false otherwise. // The object q will fly if it's balanced (it is a palindromic array) and the sum of its elements is less than or equal the maximum possible weight w. // Example: @@ -7184,7 +7184,7 @@ function will_it_fly(q, w){ const element = q[i] * -1; // console.log(""element"", element); if(!will_it_fly(element)){ -",405,0.8964 +",405,0.8964 2025-09-22T13:57:17.397733,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7214,7 +7214,7 @@ class Problem { } }",140.9568000017316,,Completions generated successfully. Ready for display in your IDE.,0201adec-8b2b-4bcd-8cc9-36a21f955854,1,deepseek-ai/deepseek-coder-1.3b-base," if (n > 1) -",115,1.0 +",115,1.0 2025-09-22T13:57:17.469341,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7262,7 +7262,7 @@ class Problem { return valid_date; } }",72.06720000249334,,Completions generated successfully. Ready for display in your IDE.,4c1cf7ec-276b-4e5d-84d2-ed0321d624e4,1,deepseek-ai/deepseek-coder-1.3b-base," -",45,1.0 +",45,1.0 2025-09-22T13:57:17.559872,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7302,7 +7302,7 @@ class Problem { } return arr.size() == 3;",},89.57910000026459,,Completions generated successfully. Ready for display in your IDE.,42c88fa0-70a4-4479-a8b3-ebdac8406bd4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:57:17.799686,[1],"from typing import List def Strongest_Extension(class_name: str, extensions: List[str]) -> str: @@ -7330,7 +7330,7 @@ def Strongest_Extension(class_name: str, extensions: List[str]) -> str: if n_uppercase - n_lowercase > best_strength:"," best_extension = extension return class_name + '.' + best_extension",239.88240000471706,,Completions generated successfully. Ready for display in your IDE.,44b87d43-8455-4187-8bcb-be37c7f93277,1,deepseek-ai/deepseek-coder-1.3b-base," best_strength = n_uppercase - n_lowercase -",214,1.0 +",214,1.0 2025-09-22T13:57:17.971772,[1],"//""Given an array representing a branch of a tree that has non-negative integer nodes // your task is to pluck one of the nodes and return it. // The plucked node should be the node with the smallest even value. @@ -7369,7 +7369,7 @@ function pluck(arr){ ",171.90010000194889,,Completions generated successfully. Ready for display in your IDE.,1e980af1-eb5d-48b6-bcdd-6156e997aa72,1,deepseek-ai/deepseek-coder-1.3b-base,"else{ continue; } - ",140,0.9549 + ",140,0.9549 2025-09-22T13:57:18.236155,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7401,7 +7401,7 @@ class Problem { } }",263.76059999893187,,Completions generated successfully. Ready for display in your IDE.,90a01308-64eb-4045-8adb-7acaddba15e8,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < game.size(); ++i) { -",238,1.0 +",238,1.0 2025-09-22T13:57:18.326928,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7439,7 +7439,7 @@ class Problem { } }",90.96979999594623,,Completions generated successfully. Ready for display in your IDE.,d96c2360-1b6e-4dbc-acb3-1a1fe4030d31,1,deepseek-ai/deepseek-coder-1.3b-base," } -",67,1.0 +",67,1.0 2025-09-22T13:57:18.682137,[1],"//Given array of integers, return array in strange order. // Strange sorting, is when you start with the minimum value, // then maximum of the remaining integers, then minimum and so on. @@ -7467,7 +7467,7 @@ function strange_sort_list(lst){ }",355.04930000024615,,Completions generated successfully. Ready for display in your IDE.,212c2b38-7779-4cb8-9eca-21cc65c2e24f,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i = 0; i < Math.floor(lst.length / 2); ++i ) { -",329,0.9856 +",329,0.9856 2025-09-22T13:57:18.789292,[1],"from typing import List def factorize(n: int) -> List[int]: @@ -7489,7 +7489,7 @@ def factorize(n: int) -> List[int]: n = n / i"," return factors ",106.8479000023217,,Completions generated successfully. Ready for display in your IDE.,43e892e2-2074-45e7-9f14-4fa49439c539,1,deepseek-ai/deepseek-coder-1.3b-base," i += 1 -",84,1.0 +",84,1.0 2025-09-22T13:57:18.879518,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7513,7 +7513,7 @@ class Problem { } }",90.00750000268454,,Completions generated successfully. Ready for display in your IDE.,07f53544-02a3-4937-8d2c-253595f475e4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:19.058221,[1],"def is_prime(n: int) -> bool: """"""Return true if a given number is prime, and false otherwise. >>> is_prime(6) @@ -7539,7 +7539,7 @@ class Problem { ",178.34160000347765,,Completions generated successfully. Ready for display in your IDE.,c3b0e34d-47db-46f2-a315-ece27368cb07,1,deepseek-ai/deepseek-coder-1.3b-base," # Your code here if n <= 1: -",155,0.947 +",155,0.947 2025-09-22T13:57:19.681965,[1],"//Find the shortest palindrome that begins with a supplied string. // Algorithm idea is simple: // - Find the longest postfix of supplied string that is a palindrome. @@ -7567,7 +7567,7 @@ function make_palindrome(string){ if (s.slice(-j) != s.slice(0, j)) return false; } return true; -",600,0.9819 +",600,0.9819 2025-09-22T13:57:19.822051,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -7592,7 +7592,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: return groups ",139.6530999991228,,Completions generated successfully. Ready for display in your IDE.,dfc966d6-2d21-4e26-b832-7f056ba0be88,1,deepseek-ai/deepseek-coder-1.3b-base," open_count -= 1 -",114,1.0 +",114,1.0 2025-09-22T13:57:20.517433,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7615,7 +7615,7 @@ class Problem { long max = numbers.get(0); List results = new ArrayList<>() {{ add(max); }}; for (int i = 1; i < numbers.size(); i++) { - Long current = numbers",670,0.9821 + Long current = numbers",670,0.9821 2025-09-22T13:57:20.720826,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7641,7 +7641,7 @@ class Problem { return x * y; } }",203.06540000456152,,Completions generated successfully. Ready for display in your IDE.,9303a6ff-2442-41c1-a4a9-b33a6f96fb9d,1,deepseek-ai/deepseek-coder-1.3b-base," long x = Math.abs(a % 10); -",178,1.0 +",178,1.0 2025-09-22T13:57:20.889999,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7665,7 +7665,7 @@ class Problem { return result; } }",169.1168999968795,,Completions generated successfully. Ready for display in your IDE.,07dbe39b-80fc-4d0f-b061-9a9aa1549e1e,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList result = new ArrayList<>(); -",145,0.9525 +",145,0.9525 2025-09-22T13:57:21.037915,[1],"//In this Kata, you have to sort an array of non-negative integers according to // number of ones in their binary representation in ascending order. // For similar number of ones, sort based on decimal value. @@ -7686,7 +7686,7 @@ function sort_array(arr){ }); }",147.50220000132686,,Completions generated successfully. Ready for display in your IDE.,29ec0c76-fe94-4c31-997c-2f7f6fe3ee20,1,deepseek-ai/deepseek-coder-1.3b-base," return aBinary - bBinary; -",123,1.0 +",123,1.0 2025-09-22T13:57:21.301258,[1],"//You are given a positive integer n. You have to create an integer array a of length n. // For each i (1 ≤ i ≤ n), the value of a[i] = i * i - i + 1. // Return the number of triples (a[i], a[j], a[k]) of a where i < j < k, @@ -7714,7 +7714,7 @@ function get_max_triples(n){ } ",263.05379999394063,,Completions generated successfully. Ready for display in your IDE.,4735bd1a-1afe-4a08-bdb9-731d3737f3ff,1,deepseek-ai/deepseek-coder-1.3b-base," for(let k = j+1; k(Arrays.asList(total, left));",},105.69789999863133,,Completions generated successfully. Ready for display in your IDE.,9fcf8146-ca12-409c-a66b-14bf99885f83,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:57:21.588407,[1],"//You are given a 2 dimensional data, as a nested arrays, // which is similar to matrix, however, unlike matrices, // each row may contain a different number of columns. @@ -7784,7 +7784,7 @@ function get_row(lst, x){ } ",180.92579999938607,,Completions generated successfully. Ready for display in your IDE.,70197b38-46ba-45eb-b74b-3c662bf922a2,1,deepseek-ai/deepseek-coder-1.3b-base," if (array[j] === x){ -",155,0.9118 +",155,0.9118 2025-09-22T13:57:21.667277,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7823,7 +7823,7 @@ class Problem { } return result; }",,79.16040000418434,,Completions generated successfully. Ready for display in your IDE.,d3240523-e173-4d58-9eb3-a69974d394ff,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:57:21.724566,[1],"from typing import List def by_length(arr: List[int]) -> List[str]: @@ -7849,7 +7849,7 @@ def by_length(arr: List[int]) -> List[str]: digits[i] = ['One', 'Two', 'Three', 'Four', 'Five', 'Six', 'Seven', 'Eight', 'Nine'][digits[i] - 1] return digits ",56.585899998026434,,Completions generated successfully. Ready for display in your IDE.,143540bf-f7d0-418f-8557-80facd25326f,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T13:57:21.906987,[1],"def anti_shuffle(s: str) -> str: """""" Write a function that takes a string and returns an ordered version of it. @@ -7869,7 +7869,7 @@ def by_length(arr: List[int]) -> List[str]: words = s.split(' ') shuffled_words = [''.join(sorted(word)) for word in words]"," ",182.41560000024037,,Completions generated successfully. Ready for display in your IDE.,91e2c29e-67ad-491b-9b6a-a30f4d04e291,1,deepseek-ai/deepseek-coder-1.3b-base," - return ' '.join(shuffled_words)",155,1.0 + return ' '.join(shuffled_words)",155,1.0 2025-09-22T13:57:22.083074,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -7899,7 +7899,7 @@ class Problem { } }",176.23720000119647,,Completions generated successfully. Ready for display in your IDE.,956214a6-e1a8-42df-9f38-70dbf3a5c9ab,1,deepseek-ai/deepseek-coder-1.3b-base," while (n % d == 0) { -",151,0.9315 +",151,0.9315 2025-09-22T13:57:22.280893,[1],"from typing import List def max_element(l: List[int]) -> int: @@ -7914,7 +7914,7 @@ def max_element(l: List[int]) -> int: return max_elem ",197.2581000009086,,Completions generated successfully. Ready for display in your IDE.,7a5742bc-3898-463b-a0b9-e1ce471c1c47,1,deepseek-ai/deepseek-coder-1.3b-base," if (l[i] > max_elem): -",164,0.9629 +",164,0.9629 2025-09-22T13:57:22.479270,[1],"//For a given string, flip lowercase characters to uppercase and uppercase to lowercase. // >>> flip_case(""Hello"") // ""hELLO"" @@ -7929,7 +7929,7 @@ function flip_case(string){ } ",198.0353999970248,,Completions generated successfully. Ready for display in your IDE.,05133b86-729a-4a47-af14-2810703d2705,1,deepseek-ai/deepseek-coder-1.3b-base," result += string[i].toUpperCase(); -",175,1.0 +",175,1.0 2025-09-22T13:57:22.543733,[1],"//You'll be given a string of words, and your task is to count the number // of boredoms. A boredom is a sentence that starts with the word ""I"". // Sentences are delimited by '.', '?' or '!'. @@ -7947,7 +7947,7 @@ function is_bored(S){ } } return boredoms;",,64.69030000152998,,Completions generated successfully. Ready for display in your IDE.,b205fbb5-2b42-4f90-99e1-afdb20a2ae9a,1,deepseek-ai/deepseek-coder-1.3b-base," -}",44,1.0 +}",44,1.0 2025-09-22T13:57:22.832050,[1],"//For a given number n, find the largest number that divides n evenly, smaller than n // >>> largest_divisor(15) // 5 @@ -7961,7 +7961,7 @@ function largest_divisor(n){ return ld; }",287.87200000078883,,Completions generated successfully. Ready for display in your IDE.,7bd80a88-378b-4f7b-a3ef-fdc3a7e62a53,1,deepseek-ai/deepseek-coder-1.3b-base," for (let i=2;i>> prime_fib(1) // 2 @@ -8048,7 +8048,7 @@ function prime_fib(n){ } return fib(fibNum - 1); }",143.59269999840762,,Completions generated successfully. Ready for display in your IDE.,db89b77a-c207-47d1-885b-626f2509836d,1,deepseek-ai/deepseek-coder-1.3b-base," return Math.abs(b); -",120,1.0 +",120,1.0 2025-09-22T13:57:23.155790,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -8125,7 +8125,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",114.05910000030417,,Completions generated successfully. Ready for display in your IDE.,cac2990f-f813-4819-a23f-aeb6cfaa7c3a,1,deepseek-ai/deepseek-coder-1.3b-base," return b -",87,1.0 +",87,1.0 2025-09-22T13:57:23.249163,[1],"//For a given string, flip lowercase characters to uppercase and uppercase to lowercase. // >>> flip_case(""Hello"") // ""hELLO"" @@ -8140,7 +8140,7 @@ function flip_case(string){ return result; } ",93.14150000136578,,Completions generated successfully. Ready for display in your IDE.,f89a4c74-d95d-497d-bae0-dc82e99dc029,1,deepseek-ai/deepseek-coder-1.3b-base," var result = """"; -",72,1.0 +",72,1.0 2025-09-22T13:57:23.396203,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -8165,7 +8165,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: return groups ",146.82750000065425,,Completions generated successfully. Ready for display in your IDE.,7f5097b9-6b9e-4547-a4a1-1eb159a78774,1,deepseek-ai/deepseek-coder-1.3b-base," if open_count == 0: -",124,1.0 +",124,1.0 2025-09-22T13:57:23.571654,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8205,7 +8205,7 @@ class Problem { return b; } }",175.8339000007254,,Completions generated successfully. Ready for display in your IDE.,5be29487-f7c6-4f12-868c-f15e27b7066d,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList b = new ArrayList<>(); -",150,1.0 +",150,1.0 2025-09-22T13:57:23.666620,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8247,7 +8247,7 @@ class Problem { } }",94.18230000301264,,Completions generated successfully. Ready for display in your IDE.,98ed9404-ee7b-4136-8a9e-7e0cbc585f62,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:57:24.220317,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8284,7 +8284,7 @@ class Problem { } }",554.0133999966201,,Completions generated successfully. Ready for display in your IDE.,cfd2148f-79f8-49ca-a5ad-3d977134c7b0,1,deepseek-ai/deepseek-coder-1.3b-base," if (txt.substring(len - 3, len - 2).matches("" "") && !txt.substring(len - 4, len - 3).matches(""\\w+"")) { -",529,0.9883 +",529,0.9883 2025-09-22T13:57:24.322721,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8316,7 +8316,7 @@ class Problem { } }",102.2923000055016,,Completions generated successfully. Ready for display in your IDE.,65eb812a-c9db-4891-9ccc-45810b0c1dc4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",76,0.9887 +",76,0.9887 2025-09-22T13:57:24.422967,[1],"//We have an array 'arr' of N integers arr[1], arr[2], ..., arr[N].The // numbers in the array will be randomly ordered. Your task is to determine if // it is possible to get an array sorted in non-decreasing order by performing @@ -8353,7 +8353,7 @@ function move_one_ball(arr){ } }",99.82389999640873,,Completions generated successfully. Ready for display in your IDE.,da934950-2848-409b-8172-1a12a476f242,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,0.9628 +",70,0.9628 2025-09-22T13:57:24.517003,[1],"//You are given a word. Your task is to find the closest vowel that stands between // two consonants from the right side of the word (case sensitive). // Vowels in the beginning and ending doesn't count. Return empty string if you didn't @@ -8391,7 +8391,7 @@ function get_closest_vowel(word){ return result[0]; }",},94.31779999431456,,Completions generated successfully. Ready for display in your IDE.,ad19c51f-6dee-43b4-8952-30dfcad67f69,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:57:24.597009,[1],"//You'll be given a string of words, and your task is to count the number // of boredoms. A boredom is a sentence that starts with the word ""I"". // Sentences are delimited by '.', '?' or '!'. @@ -8410,7 +8410,7 @@ function is_bored(S){ } ",78.9260999954422,,Completions generated successfully. Ready for display in your IDE.,50ba5dc6-41af-41b6-9e55-fa5dc4667bb6,1,deepseek-ai/deepseek-coder-1.3b-base," -",57,0.9862 +",57,0.9862 2025-09-22T13:57:24.785652,[1],"def count_upper(s: str) -> int: """""" Given a string s, count the number of uppercase vowels in even indices. @@ -8434,7 +8434,7 @@ Given a positive integer n, write a recursive function that returns the sum of t positive odd integers smaller than n. Assume n is greater than or equal to 3. """"""",188.50559999555117,,Completions generated successfully. Ready for display in your IDE.,f636dd00-643e-4542-b26c-a3b8c6bb6e0b,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(s)): -",151,1.0 +",151,1.0 2025-09-22T13:57:24.854004,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -8501,7 +8501,7 @@ function do_algebra(operator, operand){ return answer; } ",68.10129999939818,,Completions generated successfully. Ready for display in your IDE.,45de48c0-8bef-4903-945d-48ada8ccc2eb,1,deepseek-ai/deepseek-coder-1.3b-base," -",41,1.0 +",41,1.0 2025-09-22T13:57:24.923076,[1],"from typing import List, Optional def next_smallest(lst: List[int]) -> Optional[int]: @@ -8528,7 +8528,7 @@ def next_smallest(lst: List[int]) -> Optional[int]: min2 = lst[i] return min2 if min1 != min2 else None ",69.30339999962598,,Completions generated successfully. Ready for display in your IDE.,7698b5d6-db52-443e-b053-76459a54d7fd,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:57:25.069294,[1],"//Circular shift the digits of the integer x, shift the digits right by shift // and return the result as a string. // If shift > number of digits, return digits reversed. @@ -8549,7 +8549,7 @@ function circular_shift(x, shift){ } ",145.83539999875939,,Completions generated successfully. Ready for display in your IDE.,377f4f89-d7db-4d80-90ab-3b908f7144a5,1,deepseek-ai/deepseek-coder-1.3b-base," var len = str.length; -",115,1.0 +",115,1.0 2025-09-22T13:57:25.155179,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8578,7 +8578,7 @@ class Problem { } return b.toString();",},85.76210000319406,,Completions generated successfully. Ready for display in your IDE.,b4a5fce1-4f93-42f5-a8b1-e2b4eef17655,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:57:25.237980,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -8615,7 +8615,7 @@ function max_fill(grid, capacity){ } ",82.4577999956091,,Completions generated successfully. Ready for display in your IDE.,9f7dd934-a512-497c-99c5-0e991a128aa7,1,deepseek-ai/deepseek-coder-1.3b-base," -",57,1.0 +",57,1.0 2025-09-22T13:57:25.323446,[1],"//Evaluate whether the given number n can be written as the sum of exactly 4 positive even numbers // Example // >>> is_equal_to_sum_even(4) @@ -8644,7 +8644,7 @@ function is_equal_to_sum_even(n){ } }",85.70980000513373,,Completions generated successfully. Ready for display in your IDE.,d615edfc-a9b7-4a49-93d9-80d1f8b3345c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:57:25.434455,[1],"//Write a function that takes a message, and encodes in such a // way that it swaps case of all letters, replaces all vowels in // the message with the letter that appears 2 places ahead of that @@ -8667,7 +8667,7 @@ function encode(message){ result += letter.toUpperCase() === letter ? letter.toLowerCase() : letter.toUpperCase(); }",},110.33820000011474,,Completions generated successfully. Ready for display in your IDE.,2a55495d-4e56-4075-9706-4ed63ee03a96,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",85,0.8848 +",85,0.8848 2025-09-22T13:57:25.562812,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8709,7 +8709,7 @@ class Problem { } }",128.34419999853708,,Completions generated successfully. Ready for display in your IDE.,68d33548-7084-4bd2-966d-0298c8157016,1,deepseek-ai/deepseek-coder-1.3b-base," String strongest = """"; -",100,1.0 +",100,1.0 2025-09-22T13:57:25.813028,[1],"//Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. // >>> fizz_buzz(50) // 0 @@ -8731,7 +8731,7 @@ function fizz_buzz(n){ } ",249.63170000410173,,Completions generated successfully. Ready for display in your IDE.,22efff65-ca5d-4d3e-aa87-2a3a15e20752,1,deepseek-ai/deepseek-coder-1.3b-base," for (var j=0; j 0) { -",128,0.9712 +",128,0.9712 2025-09-22T13:57:26.138097,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8841,7 +8841,7 @@ class Problem { }",159.4175999998697,,Completions generated successfully. Ready for display in your IDE.,b3c60469-299f-4dc2-aa9b-c6764c99d7c5,1,deepseek-ai/deepseek-coder-1.3b-base," { return null; } -",134,0.9615 +",134,0.9615 2025-09-22T13:57:26.295981,[1],"from typing import Tuple def even_odd_count(num: int) -> Tuple[int, int]: @@ -8864,7 +8864,7 @@ def even_odd_count(num: int) -> Tuple[int, int]: ",157.66230000008363,,Completions generated successfully. Ready for display in your IDE.,62241295-90ee-4e50-b2fd-55a936850f22,1,deepseek-ai/deepseek-coder-1.3b-base," if digit % 2 == 0: -",134,1.0 +",134,1.0 2025-09-22T13:57:26.374527,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8906,7 +8906,7 @@ class Problem { } return class_name + ""."" + strongest; }",,78.19290000043111,,Completions generated successfully. Ready for display in your IDE.,857377f4-c8f4-487c-bd02-39dc6b213411,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:57:26.436772,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -8943,7 +8943,7 @@ function even_odd_palindrome(n){ } return [even, odd]; }",61.81759999890346,,Completions generated successfully. Ready for display in your IDE.,2206614a-946b-4b69-bb54-d0bbb6518fbe,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T13:57:26.634211,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -8986,7 +8986,7 @@ class Problem { } }",197.6832000000286,,Completions generated successfully. Ready for display in your IDE.,b7659887-1e3d-40ee-8d15-79f26dd0a2e4,1,deepseek-ai/deepseek-coder-1.3b-base," if (2 * x + y < n) { -",167,1.0 +",167,1.0 2025-09-22T13:57:26.887778,[1],"from typing import List def maximum(arr: List[int], k: int) -> List[int]: @@ -9027,7 +9027,7 @@ def maximum(arr: List[int], k: int) -> List[int]: arr.remove(max_num)"," return sorted(get_max(arr, k)) ",253.53549999999814,,Completions generated successfully. Ready for display in your IDE.,e4a51425-fad5-4950-ac15-d3330fc2f43a,1,deepseek-ai/deepseek-coder-1.3b-base," - return [max_num] + get_max(arr, k - 1)",230,0.9884 + return [max_num] + get_max(arr, k - 1)",230,0.9884 2025-09-22T13:57:26.950736,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -9076,7 +9076,7 @@ function prime_fib(n){ } return fib(fibNum - 1); }",62.18970000190893,,Completions generated successfully. Ready for display in your IDE.,8f1c3c7b-53f6-4f6a-ad52-a291fb41225f,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:57:27.089311,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -9105,7 +9105,7 @@ function f(n){ return result; }",138.60169999679783,,Completions generated successfully. Ready for display in your IDE.,10671cd2-1c9d-4b77-959b-e8d1d328dc85,1,deepseek-ai/deepseek-coder-1.3b-base," factorial *= j; -",114,0.9897 +",114,0.9897 2025-09-22T13:57:27.223654,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -9145,7 +9145,7 @@ function match_parens(lst){ return 'No'; }",134.448999997403,,Completions generated successfully. Ready for display in your IDE.,d13390c8-622c-4681-b6df-031fd6444c87,1,deepseek-ai/deepseek-coder-1.3b-base," const close = ')'; -",109,1.0 +",109,1.0 2025-09-22T13:57:27.357748,[1],"def is_palindrome(text: str) -> bool: """""" Checks if given string is a palindrome @@ -9164,7 +9164,7 @@ function match_parens(lst){ return True ",133.54480000270996,,Completions generated successfully. Ready for display in your IDE.,0bf26430-da85-41db-bd60-fbe802b3ca06,1,deepseek-ai/deepseek-coder-1.3b-base," length = len(text) -",108,1.0 +",108,1.0 2025-09-22T13:57:27.487851,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -9196,7 +9196,7 @@ class Problem { } }",130.06730000051903,,Completions generated successfully. Ready for display in your IDE.,6089f621-78d3-41bf-8aa8-7f44f582fa62,1,deepseek-ai/deepseek-coder-1.3b-base," boolean found = false; -",105,1.0 +",105,1.0 2025-09-22T13:57:27.600741,[1],"//Given an array of numbers, return whether or not they are sorted // in ascending order. If array has more than 1 duplicate of the same // number, return false. Assume no negative numbers and only integers. @@ -9238,7 +9238,7 @@ function is_sorted(lst){ }","} ",112.41700000391575,,Completions generated successfully. Ready for display in your IDE.,75c9c10a-6d25-4d57-85dc-d4c0dfa741b0,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",89,1.0 +",89,1.0 2025-09-22T13:57:27.700325,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -9270,7 +9270,7 @@ function correct_bracketing(brackets){ } }",99.86550000030547,,Completions generated successfully. Ready for display in your IDE.,84d41db9-7435-47db-889f-37e0f5e088e7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,0.9606 +",61,0.9606 2025-09-22T13:57:27.816856,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -9307,7 +9307,7 @@ class Problem { }"," } }",116.15390000224579,,Completions generated successfully. Ready for display in your IDE.,3465d9ca-c9da-4b9e-9029-734c7d4c9d93,1,deepseek-ai/deepseek-coder-1.3b-base," return output; -",89,0.9922 +",89,0.9922 2025-09-22T13:57:28.270817,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -9342,7 +9342,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",453.35979999799747,,Completions generated successfully. Ready for display in your IDE.,380ba5ba-f061-413b-8930-42b87c5861c6,1,deepseek-ai/deepseek-coder-1.3b-base," if abs(numbers[i] - numbers[j]) < temp: if abs(numbers[i] - numbers[j]) < temp2: -",425,0.9872 +",425,0.9872 2025-09-22T13:57:28.328759,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -9379,7 +9379,7 @@ function is_prime(n){ return true; } ",58.02109999785898,,Completions generated successfully. Ready for display in your IDE.,33b94dbe-dd8f-48de-8b0c-906a121eb882,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:57:28.508859,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -9457,7 +9457,7 @@ class Problem { } }",179.80510000052163,,Completions generated successfully. Ready for display in your IDE.,9737433a-a650-4547-b452-64cbc537302a,1,deepseek-ai/deepseek-coder-1.3b-base," if (num >= 900) { -",154,1.0 +",154,1.0 2025-09-22T13:57:28.635771,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -9506,7 +9506,7 @@ function check_dict_case(dict){ return true; }",126.52830000297399,,Completions generated successfully. Ready for display in your IDE.,71713430-47d1-4a46-be1c-929d56dc9d5b,1,deepseek-ai/deepseek-coder-1.3b-base," else { -",85,1.0 +",85,1.0 2025-09-22T13:57:29.089477,[1],"from typing import List def minPath(grid: List[List[int]], k: int) -> List[int]: @@ -9554,7 +9554,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: path.append(min_val) return path",454.04270000290126,,Completions generated successfully. Ready for display in your IDE.,ca94eccd-c7ce-47a5-85aa-7bc22a41fae6,1,deepseek-ai/deepseek-coder-1.3b-base," for i, j in [(row-1, col), (row+1, col), (row, col-1), (row, col+1)]: -",425,0.9872 +",425,0.9872 2025-09-22T13:57:29.190492,[1],"from typing import List def by_length(arr: List[int]) -> List[str]: @@ -9580,7 +9580,7 @@ def by_length(arr: List[int]) -> List[str]: digits.reverse() for i in range(len(digits)): digits[i] = ['One', 'Two', 'Three', 'Four', 'Five', 'Six', 'Seven', 'Eight', 'Nine'][digits[i] - 1]",,100.18550000677351,,Completions generated successfully. Ready for display in your IDE.,0741bac3-0410-4f40-a6d8-d68f89c795cb,1,deepseek-ai/deepseek-coder-1.3b-base," - return digits",74,0.9866 + return digits",74,0.9866 2025-09-22T13:57:29.338034,[1],"//"" // This function will take an array of integers. For all entries in the array, the function shall square the integer entry if its index is a // multiple of 3 and will cube the integer entry if its index is a multiple of 4 and not a multiple of 3. The function will not @@ -9605,7 +9605,7 @@ function sum_squares(lst){ return sum; }",147.57059999828925,,Completions generated successfully. Ready for display in your IDE.,ce2ab9e2-3b60-4cf5-8dda-ec0e2ecf7b86,1,deepseek-ai/deepseek-coder-1.3b-base," var len = lst.length; -",124,1.0 +",124,1.0 2025-09-22T13:57:29.429895,[1],"//Task // We are given two strings s and c, you have to deleted all the characters in s that are equal to any character in c // then check if the result string is palindrome. @@ -9629,7 +9629,7 @@ function reverse_delete(s, c){ result.push(filterString, false);"," return result; }",91.21929999673739,,Completions generated successfully. Ready for display in your IDE.,84b851e1-fb11-498c-a122-b9056d4f57bc,1,deepseek-ai/deepseek-coder-1.3b-base," } -",66,1.0 +",66,1.0 2025-09-22T13:57:29.500862,[1],"//Given a string s, count the number of uppercase vowels in even indices. // For example: // >>> count_upper(""aBCdEf"") @@ -9648,7 +9648,7 @@ function count_upper(s){ } } return result;",,71.11849999637343,,Completions generated successfully. Ready for display in your IDE.,832b13c2-12ff-44d0-bb46-40b343d03d3c,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T13:57:29.610073,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -9690,7 +9690,7 @@ function compare_one(a, b){ } ",108.963999999105,,Completions generated successfully. Ready for display in your IDE.,9b140258-0ff5-4fbb-b336-2bdf3c362646,1,deepseek-ai/deepseek-coder-1.3b-base," return a; -",84,1.0 +",84,1.0 2025-09-22T13:57:29.705084,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -9730,7 +9730,7 @@ class Problem { }); return words.get(0);",},94.53470000153175,,Completions generated successfully. Ready for display in your IDE.,d3a44206-2d0c-4b35-aaf8-2353773d2c6d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:57:29.821849,[1],"def correct_bracketing(brackets: str) -> bool: """""" brackets is a string of ""("" and "")"". return True if every opening bracket has a corresponding closing bracket. @@ -9754,7 +9754,7 @@ class Problem { ",116.54149999958463,,Completions generated successfully. Ready for display in your IDE.,31aebd60-0232-40d8-843d-3286ed9c99dc,1,deepseek-ai/deepseek-coder-1.3b-base," count -= 1 -",93,1.0 +",93,1.0 2025-09-22T13:57:29.938994,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -9786,7 +9786,7 @@ class Problem { } }",117.01529999845661,,Completions generated successfully. Ready for display in your IDE.,d4bf9d84-e30d-4d30-af5e-5434c2b85e7a,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",88,1.0 +",88,1.0 2025-09-22T13:57:30.137045,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -9863,7 +9863,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",198.34980000450741,,Completions generated successfully. Ready for display in your IDE.,e930c834-0a25-4891-8394-65493488c186,1,deepseek-ai/deepseek-coder-1.3b-base," if float(a_tmp) > b: -",170,1.0 +",170,1.0 2025-09-22T13:57:30.289828,[1],"//There are eight planets in our solar system: the closerst to the Sun // is Mercury, the next one is Venus, then Earth, Mars, Jupiter, Saturn, // Uranus, Neptune. @@ -9907,7 +9907,7 @@ function bf(planet1, planet2){ return res; }",152.6942999989842,,Completions generated successfully. Ready for display in your IDE.,090042a7-ada8-4cc4-a646-dceb1a557202,1,deepseek-ai/deepseek-coder-1.3b-base," let temp = position1; -",120,0.9956 +",120,0.9956 2025-09-22T13:57:30.432780,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -9985,7 +9985,7 @@ class Problem { } }",142.35139999800595,,Completions generated successfully. Ready for display in your IDE.,e58e038b-15d9-4c70-a571-8edf0ab8bb3b,1,deepseek-ai/deepseek-coder-1.3b-base," res += ""xc""; -",113,1.0 +",113,1.0 2025-09-22T13:57:31.137703,[1],"//Input to this function is a string representing musical notes in a special ASCII format. // Your task is to parse this string and return array of integers corresponding to how many beats does each // not last. @@ -10013,7 +10013,7 @@ function parse_music(music_string){ if (isNaN(Number(s))) { switch (s) { case ""o"": - song.push",668,0.9548 + song.push",668,0.9548 2025-09-22T13:57:31.251251,[1],"//Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. // >>> fizz_buzz(50) // 0 @@ -10035,7 +10035,7 @@ function fizz_buzz(n){ }","} ",113.78700000204844,,Completions generated successfully. Ready for display in your IDE.,dea52330-2256-40ce-8257-4b660544515b,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",90,1.0 +",90,1.0 2025-09-22T13:57:31.478134,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10079,7 +10079,7 @@ class Problem { } }",226.05060000205413,,Completions generated successfully. Ready for display in your IDE.,19fa7520-baf8-476d-b4d9-f750417ec6b6,1,deepseek-ai/deepseek-coder-1.3b-base," else if (op.get(i).equals(""*"")) -",199,1.0 +",199,1.0 2025-09-22T13:57:31.839934,[1],"//From a given array of integers, generate an array of rolling maximum element found until given moment // in the sequence. // >>> rolling_max([1, 2, 3, 2, 3, 4, 2]) @@ -10097,7 +10097,7 @@ function rolling_max(numbers){ tempMax = Math.max(tempMax, numbers[i]); } max.push(tempMax); -",339,0.9344 +",339,0.9344 2025-09-22T13:57:31.923867,[1],"//triples_sum_to_zero takes an array of integers as an input. // it returns true if there are three distinct elements in the array that // sum to zero, and false otherwise. @@ -10125,7 +10125,7 @@ function triples_sum_to_zero(l){ } ",84.09209999808809,,Completions generated successfully. Ready for display in your IDE.,9592b9a7-3978-48d7-bef4-fc0389f3001d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:57:32.011647,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10154,7 +10154,7 @@ class Problem { if (n == 3) return 0; return fib4(n-1) + fib4(n-2) + fib4(n-3) + fib4(n-4);",},87.48880000348436,,Completions generated successfully. Ready for display in your IDE.,0cf7a87d-92af-465d-99d2-feb2fdfdaf4a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:57:32.055776,[1],"def is_multiply_prime(a: int) -> bool: """"""Write a function that returns true if the given number is the multiplication of 3 prime numbers and false otherwise. @@ -10179,7 +10179,7 @@ class Problem { else: return False -",44.04920000524726,,Completions generated successfully. Ready for display in your IDE.,ae0884cb-e92a-48b8-9ae4-cb1bc6efcd52,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",44.04920000524726,,Completions generated successfully. Ready for display in your IDE.,ae0884cb-e92a-48b8-9ae4-cb1bc6efcd52,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:57:32.195178,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10230,7 +10230,7 @@ class Problem { return ""Yes""; } }",138.54100000025937,,Completions generated successfully. Ready for display in your IDE.,b3882d2b-afa2-40fb-8230-b01e32914b7c,1,deepseek-ai/deepseek-coder-1.3b-base," String result = ""No""; -",111,1.0 +",111,1.0 2025-09-22T13:57:32.344436,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10283,7 +10283,7 @@ class Problem { newArr.add((long)minIndex);"," } }",149.72470000066096,,Completions generated successfully. Ready for display in your IDE.,6b508cf0-f53e-4808-9455-7fd32ff43c29,1,deepseek-ai/deepseek-coder-1.3b-base," return newArr; -",121,1.0 +",121,1.0 2025-09-22T13:57:32.543683,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10322,7 +10322,7 @@ class Problem { return res; } }",198.70369999989634,,Completions generated successfully. Ready for display in your IDE.,2dbee4fc-af82-418c-8747-1d7b60ff3619,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList res = new ArrayList(); -",174,1.0 +",174,1.0 2025-09-22T13:57:32.638333,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -10361,7 +10361,7 @@ function find_max(words){ return max_unique_word; }",94.81599999708124,,Completions generated successfully. Ready for display in your IDE.,14f68593-099d-453b-9088-00842445ff61,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:57:32.818044,[1],"//Write a function that accepts two arrays of strings and returns the array that has // total number of chars in the all strings of the array less than the other array. // if the two arrays have the same number of chars, return the first array. @@ -10392,7 +10392,7 @@ function total_match(lst1, lst2){ } ",179.1808999987552,,Completions generated successfully. Ready for display in your IDE.,9a8b1fc7-1c40-44e6-87b7-c6f420a27281,1,deepseek-ai/deepseek-coder-1.3b-base," let sum_lst2 = 0; -",150,1.0 +",150,1.0 2025-09-22T13:57:32.924797,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10441,7 +10441,7 @@ class Problem { } }",106.6011000002618,,Completions generated successfully. Ready for display in your IDE.,ccca4d7f-c8fd-4428-a882-f4bfda5b64e9,1,deepseek-ai/deepseek-coder-1.3b-base," } -",73,1.0 +",73,1.0 2025-09-22T13:57:33.047103,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -10518,7 +10518,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",121.773500002746,,Completions generated successfully. Ready for display in your IDE.,d9bb14db-cb34-42f4-a427-d7ec48c4587a,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",88,1.0 +",88,1.0 2025-09-22T13:57:33.113765,[1],"//Return array with elements incremented by 1. // >>> incr_list([1, 2, 3]) // [2, 3, 4] @@ -10530,7 +10530,7 @@ function incr_list(l){ new_list.push(l[i] + 1); } return new_list;",,67.1537999951397,,Completions generated successfully. Ready for display in your IDE.,a491cf90-cbad-4466-bf97-f511edb746e0,1,deepseek-ai/deepseek-coder-1.3b-base," -}",46,1.0 +}",46,1.0 2025-09-22T13:57:33.208797,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -10579,7 +10579,7 @@ function check_dict_case(dict){ return true; }",94.46890000253916,,Completions generated successfully. Ready for display in your IDE.,135ab0f0-efe0-4006-90ad-92eded65f1a4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",67,1.0 +",67,1.0 2025-09-22T13:57:33.902810,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -10609,7 +10609,7 @@ function vowels_count(s){ } console.log('VOWELS COUNT'); console.log (vowels_count ('abcde')); //should return 2 -console.",667,0.9079 +console.",667,0.9079 2025-09-22T13:57:33.998191,[1],"//"" // This function will take an array of integers. For all entries in the array, the function shall square the integer entry if its index is a // multiple of 3 and will cube the integer entry if its index is a multiple of 4 and not a multiple of 3. The function will not @@ -10634,7 +10634,7 @@ function sum_squares(lst){ } }",},94.67959999892628,,Completions generated successfully. Ready for display in your IDE.,1127e649-339d-4209-ab78-006e366a97ad,1,deepseek-ai/deepseek-coder-1.3b-base," return sum; -",72,1.0 +",72,1.0 2025-09-22T13:57:34.704708,[1],"from typing import List def pluck(arr: List[int]) -> List[int]: @@ -10681,7 +10681,7 @@ def pluck(arr: List[int]) -> List[int]: for i, e in enumerate(evens): if e == min_even: if min_index is not None: - if arr[min_index] > arr[i]:",679,0.9647 + if arr[min_index] > arr[i]:",679,0.9647 2025-09-22T13:57:35.102508,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -10727,7 +10727,7 @@ function histogram(test){ //console.log('max', max) for (var i=0 ; i < lst.length ; ++i ) { -",372,0.9213 +",372,0.9213 2025-09-22T13:57:35.220389,[1],"//xs represent coefficients of a polynomial. // xs[0] + xs[1] * x + xs[2] * x^2 + .... // Return derivative of this polynomial in the same form. @@ -10742,7 +10742,7 @@ function derivative(xs){ }","} ",116.87019999953918,,Completions generated successfully. Ready for display in your IDE.,18089e30-5c24-4a0d-b9b2-d621eac6f776,1,deepseek-ai/deepseek-coder-1.3b-base," return coeff; -",93,0.9915 +",93,0.9915 2025-09-22T13:57:35.348493,[1],"from typing import List def strange_sort_list(lst: List[int]) -> List[int]: @@ -10766,7 +10766,7 @@ def strange_sort_list(lst: List[int]) -> List[int]: out.append(lst.pop()) return out",128.48939999821596,,Completions generated successfully. Ready for display in your IDE.,438a2518-636d-4a04-86be-83bd875f3620,1,deepseek-ai/deepseek-coder-1.3b-base," if not lst: -",104,1.0 +",104,1.0 2025-09-22T13:57:35.463439,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10803,7 +10803,7 @@ class Problem { } }",114.80079999455484,,Completions generated successfully. Ready for display in your IDE.,4b9dace2-7492-4803-b8dd-61e5f363d7c1,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",91,1.0 +",91,1.0 2025-09-22T13:57:35.735598,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10825,7 +10825,7 @@ class Problem { BigInteger nSquared = nBigInteger.multiply(nBigInteger);"," } }",271.9084999989718,,Completions generated successfully. Ready for display in your IDE.,35292fcf-1c96-4abc-b494-a77d267ffb9f,1,deepseek-ai/deepseek-coder-1.3b-base," return nSquared.divide(new BigInteger(""2"")).longValue(); -",248,1.0 +",248,1.0 2025-09-22T13:57:35.986174,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -10897,7 +10897,7 @@ function minPath(grid, k){ return path; }",250.50129999726778,,Completions generated successfully. Ready for display in your IDE.,182adbc7-1666-44bc-956b-f61337f169bc,1,deepseek-ai/deepseek-coder-1.3b-base," if(!path || !startCell && curr >= startCell){ -",219,0.9666 +",219,0.9666 2025-09-22T13:57:36.072649,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -10917,7 +10917,7 @@ class Problem { } return 1;",},85.35300000221469,,Completions generated successfully. Ready for display in your IDE.,96560824-a38c-4b63-a5cc-2a023f3b8870,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:36.115688,[1],"def make_palindrome(string: str) -> str: """""" Find the shortest palindrome that begins with a supplied string. Algorithm idea is simple: @@ -10939,7 +10939,7 @@ class Problem { if string[i:] == prefix[:len(string) - i]: return string + prefix[len(string) - i:] "," -",42.84209999605082,,Completions generated successfully. Ready for display in your IDE.,285c2546-3cca-45da-ad1d-e7558bd6b2c5,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",42.84209999605082,,Completions generated successfully. Ready for display in your IDE.,285c2546-3cca-45da-ad1d-e7558bd6b2c5,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:57:36.221367,[1],"from typing import List def pluck(arr: List[int]) -> List[int]: @@ -10982,7 +10982,7 @@ def pluck(arr: List[int]) -> List[int]: return [min(evens), arr.index(min(evens))] ",105.39699999935692,,Completions generated successfully. Ready for display in your IDE.,b8bc0f63-99ee-4c52-9aef-13c79b6dbe74,1,deepseek-ai/deepseek-coder-1.3b-base," return [] -",78,1.0 +",78,1.0 2025-09-22T13:57:36.309459,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11011,7 +11011,7 @@ class Problem { } }",87.98830000159796,,Completions generated successfully. Ready for display in your IDE.,0bc41766-52c5-43aa-8486-b70895b1012e,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,0.9493 +",62,0.9493 2025-09-22T13:57:36.569523,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11042,7 +11042,7 @@ class Problem { } }",259.99310000042897,,Completions generated successfully. Ready for display in your IDE.,b2ae10b5-ebdd-427f-bc75-d6de37cece3e,1,deepseek-ai/deepseek-coder-1.3b-base," int n_num = Integer.parseInt(n_nums[0]); -",234,1.0 +",234,1.0 2025-09-22T13:57:36.662023,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11077,7 +11077,7 @@ class Problem { } }",92.47750000213273,,Completions generated successfully. Ready for display in your IDE.,db4b401b-85c3-4fd9-bbb5-3a13e64e15ce,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:57:36.819909,[1],"def x_or_y(n: int, x: int, y: int) -> int: """"""A simple program which should return the value of x if n is a prime number and should return the value of y otherwise. @@ -11096,7 +11096,7 @@ class Problem { return x ",157.34779999911552,,Completions generated successfully. Ready for display in your IDE.,63b82da1-64cc-422c-96af-ba5bb6c0014c,1,deepseek-ai/deepseek-coder-1.3b-base," if n == 1 or n == 2: -",137,0.9841 +",137,0.9841 2025-09-22T13:57:36.968140,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -11168,7 +11168,7 @@ function minPath(grid, k){ return path; }",147.83929999975953,,Completions generated successfully. Ready for display in your IDE.,98f57472-893c-4ab7-9a71-635b2523e54a,1,deepseek-ai/deepseek-coder-1.3b-base," x = i; -",114,1.0 +",114,1.0 2025-09-22T13:57:37.075328,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11191,7 +11191,7 @@ class Problem { } }",106.95600000326522,,Completions generated successfully. Ready for display in your IDE.,2ce3a5f2-6b04-4004-a115-8e4662e418e1,1,deepseek-ai/deepseek-coder-1.3b-base," return a; -",84,1.0 +",84,1.0 2025-09-22T13:57:37.304417,[1],"def prime_length(string: str) -> bool: """"""Write a function that takes a string and returns True if the string length is a prime number or False otherwise @@ -11216,7 +11216,7 @@ class Problem { ",229.196200001752,,Completions generated successfully. Ready for display in your IDE.,3d4d0259-a049-4b81-9996-8a8a4a9656be,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(2, int(num / 2)): -",206,0.9503 +",206,0.9503 2025-09-22T13:57:37.358933,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -11245,7 +11245,7 @@ function parse_nested_parens(paren_string){ } ",54.260299999441486,,Completions generated successfully. Ready for display in your IDE.,7e475540-f63e-4c3f-b4f5-b5827bab2eb6,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T13:57:37.575797,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11283,7 +11283,7 @@ class Problem { } }",216.45530000387225,,Completions generated successfully. Ready for display in your IDE.,6ca3044e-1a91-4113-a6c5-3d18f179a210,1,deepseek-ai/deepseek-coder-1.3b-base," char[] chars = music_string.toCharArray(); -",192,1.0 +",192,1.0 2025-09-22T13:57:37.967111,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11318,7 +11318,7 @@ class Problem { } }",391.36149999831105,,Completions generated successfully. Ready for display in your IDE.,b27a95f8-597c-45ee-98f0-50439c0b120f,1,deepseek-ai/deepseek-coder-1.3b-base," lst.set(i, lst.get(i) * lst.get(i) * lst.get(i)); -",366,1.0 +",366,1.0 2025-09-22T13:57:38.105971,[1],"def correct_bracketing(brackets: str) -> bool: """""" brackets is a string of ""("" and "")"". return True if every opening bracket has a corresponding closing bracket. @@ -11342,7 +11342,7 @@ class Problem { ",138.46879999618977,,Completions generated successfully. Ready for display in your IDE.,2539fee9-005c-43a8-ab09-6f4eae9fa912,1,deepseek-ai/deepseek-coder-1.3b-base," if char == '(': -",116,1.0 +",116,1.0 2025-09-22T13:57:38.203426,[1],"//Filter given array of any jsthon values only for integers // >>> filter_integers([""a"", 3.14, 5]) // [5] @@ -11357,7 +11357,7 @@ function filter_integers(values){ });","} ",97.34020000178134,,Completions generated successfully. Ready for display in your IDE.,f7a367ca-cdc4-454c-ba29-add6e861fcfe,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",73,1.0 +",73,1.0 2025-09-22T13:57:38.349617,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11393,7 +11393,7 @@ class Problem { } }",146.31880000524689,,Completions generated successfully. Ready for display in your IDE.,f440d873-cfac-4715-95cc-5f3dfb0c8c34,1,deepseek-ai/deepseek-coder-1.3b-base," long max = 0L; -",120,0.8516 +",120,0.8516 2025-09-22T13:57:38.415179,[1],"//Given an array of numbers, return the sum of squares of the numbers // in the array that are odd. Ignore numbers that are negative or not integers. // >>> double_the_difference([1, 3, 2, 0]) @@ -11413,7 +11413,7 @@ function double_the_difference(lst){ return sum; } ",64.90409999969415,,Completions generated successfully. Ready for display in your IDE.,ba678e97-69bd-4b91-ae62-55136634bd08,1,deepseek-ai/deepseek-coder-1.3b-base," - ",41,1.0 + ",41,1.0 2025-09-22T13:57:38.504405,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11456,7 +11456,7 @@ class Problem { } }",88.79719999822555,,Completions generated successfully. Ready for display in your IDE.,fa33165e-5391-45ae-ac78-3492196f9bdf,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,0.9169 +",64,0.9169 2025-09-22T13:57:38.622865,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11482,7 +11482,7 @@ class Problem { } }",118.54229999880772,,Completions generated successfully. Ready for display in your IDE.,c7a76f67-c2fc-4fbb-aa53-707c0349c252,1,deepseek-ai/deepseek-coder-1.3b-base," chosen = i; -",93,1.0 +",93,1.0 2025-09-22T13:57:38.916038,[1],"//Input to this function is a string containing multiple groups of nested parentheses. Your goal is to // separate those group into separate strings and return the array of those. // Separate groups are balanced (each open brace is properly closed) and not nested within each other @@ -11509,7 +11509,7 @@ function separate_paren_groups(paren_string){ ",292.9230000008829,,Completions generated successfully. Ready for display in your IDE.,40a8f66f-8b0d-4592-bd92-6346d029c015,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=0; i List[int]: @@ -11555,7 +11555,7 @@ def get_positive(l: List[int]) -> List[int]: return p ",123.42699999862816,,Completions generated successfully. Ready for display in your IDE.,21b05f12-82a2-4580-8564-32ecc3d00a4e,1,deepseek-ai/deepseek-coder-1.3b-base," for i in l: -",102,1.0 +",102,1.0 2025-09-22T13:57:39.355640,[1],"//For a given array of integers, return an array consisting of a sum and a product of all the integers in an array. // Empty sum should be equal to 0 and empty product should be equal to 1. // >>> sum_product([]) @@ -11574,7 +11574,7 @@ function sum_product(numbers){ return [sum, product]; } ",229.40229999949224,,Completions generated successfully. Ready for display in your IDE.,044eebe4-2f90-41f0-913b-07ee07445aa7,1,deepseek-ai/deepseek-coder-1.3b-base," if (numbers === [] || !Array.isArray(numbers)){ -",206,0.958 +",206,0.958 2025-09-22T13:57:39.504649,[1],"//There are eight planets in our solar system: the closerst to the Sun // is Mercury, the next one is Venus, then Earth, Mars, Jupiter, Saturn, // Uranus, Neptune. @@ -11618,7 +11618,7 @@ function bf(planet1, planet2){ return res; }",148.7769999948796,,Completions generated successfully. Ready for display in your IDE.,b934bca5-3112-4599-a058-c2c1371b5dbd,1,deepseek-ai/deepseek-coder-1.3b-base," 'Neptune':8 -",123,1.0 +",123,1.0 2025-09-22T13:57:39.738888,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -11659,7 +11659,7 @@ function match_parens(lst){ }",234.08780000318075,,Completions generated successfully. Ready for display in your IDE.,9f13c3f0-6155-4de6-a1ac-59060402b327,1,deepseek-ai/deepseek-coder-1.3b-base," // your code here... const str1 = lst[0]; -",198,0.9844 +",198,0.9844 2025-09-22T13:57:39.803731,[1],"//Find how many times a given substring can be found in the original string. Count overlaping cases. // >>> how_many_times("""", ""a"") // 0 @@ -11675,7 +11675,7 @@ function how_many_times(string, substring){ i = string.indexOf(substring, i) + 1; } return count;",,65.19850000040606,,Completions generated successfully. Ready for display in your IDE.,7a37e4c4-f021-4ade-a87e-63db06a3702b,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:57:39.891810,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11703,7 +11703,7 @@ class Problem { public static long digitSum(String s) { return s.chars().filter(i -> i >= 65 && i <= 90).sum();",},87.61650000087684,,Completions generated successfully. Ready for display in your IDE.,185cd0ae-ae57-46fc-b73d-a4796fc84998,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:57:40.045044,[1],"//From a supplied array of numbers (of length at least two) select and return two that are the closest to each // other and return them in order (smaller number, larger number). // >>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) @@ -11725,7 +11725,7 @@ function find_closest_elements(numbers){ } ",152.44019999954617,,Completions generated successfully. Ready for display in your IDE.,0f1485ee-d806-4952-8f8a-3a4b5f753f02,1,deepseek-ai/deepseek-coder-1.3b-base," let b = numbers[1]; -",124,1.0 +",124,1.0 2025-09-22T13:57:40.293777,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -11798,7 +11798,7 @@ function minPath(grid, k){ }",248.63899999763817,,Completions generated successfully. Ready for display in your IDE.,d333614a-f12e-4d78-967c-d6084c31e476,1,deepseek-ai/deepseek-coder-1.3b-base," //find starting point let startCell = Infinity; -",218,0.9958 +",218,0.9958 2025-09-22T13:57:40.706279,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11838,7 +11838,7 @@ class Problem { } }",412.166400004935,,Completions generated successfully. Ready for display in your IDE.,0d052b62-2e99-4410-aeaa-b246f6738405,1,deepseek-ai/deepseek-coder-1.3b-base," memo[i] = memo[i - 1] + memo[i - 2] + memo[i - 3]; -",387,1.0 +",387,1.0 2025-09-22T13:57:40.833570,[1],"//You are given a string representing a sentence, // the sentence contains some words separated by a space, // and you have to return a string that contains the words from the original sentence, @@ -11868,7 +11868,7 @@ function words_in_sentence(sentence){ }"," }).join(' '); }",127.12800000008428,,Completions generated successfully. Ready for display in your IDE.,88b865be-e8c4-4ab1-9628-7f043494ea05,1,deepseek-ai/deepseek-coder-1.3b-base," return isPrime; -",103,1.0 +",103,1.0 2025-09-22T13:57:40.927936,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11900,7 +11900,7 @@ class Problem { } return result;",},93.97840000019642,,Completions generated successfully. Ready for display in your IDE.,40f20329-a8fb-472c-a6ef-42f0100e6f9a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:57:41.033591,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -11932,7 +11932,7 @@ class Problem { } }",105.72959999990417,,Completions generated successfully. Ready for display in your IDE.,a6406e38-f822-478b-9db8-5fb47ed2daea,1,deepseek-ai/deepseek-coder-1.3b-base," continue; -",79,0.9629 +",79,0.9629 2025-09-22T13:57:41.239472,[1],"from typing import List, Any def filter_integers(values: List[Any]) -> List[int]: @@ -11942,7 +11942,7 @@ def filter_integers(values: List[Any]) -> List[int]: >>> filter_integers([1, 2, 3, 'abc', { }, []]) [1, 2, 3] """""" -",,205.68839999759803,,Completions generated successfully. Ready for display in your IDE.,f7b3f392-fffe-4d66-b3d8-dc8f2c7b1160,1,deepseek-ai/deepseek-coder-1.3b-base," return [value for value in values if isinstance(value, int)]",184,0.9515 +",,205.68839999759803,,Completions generated successfully. Ready for display in your IDE.,f7b3f392-fffe-4d66-b3d8-dc8f2c7b1160,1,deepseek-ai/deepseek-coder-1.3b-base," return [value for value in values if isinstance(value, int)]",184,0.9515 2025-09-22T13:57:41.398157,[1],"from typing import List def sort_even(l: List[int]) -> List[int]: @@ -11961,7 +11961,7 @@ def sort_even(l: List[int]) -> List[int]: l[i] = temp[i // 2] return l",158.19879999617115,,Completions generated successfully. Ready for display in your IDE.,ba19b971-4494-4f27-8621-af507027f5d9,1,deepseek-ai/deepseek-coder-1.3b-base," temp.append(l[i]) -",134,0.9946 +",134,0.9946 2025-09-22T13:57:41.459705,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -12004,7 +12004,7 @@ function histogram(test){ return result; } ",61.33199999749195,,Completions generated successfully. Ready for display in your IDE.,f668da9f-f892-4939-b8ad-04cf44a5e8c4,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:57:41.697691,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12026,7 +12026,7 @@ class Problem { } }",238.3031000063056,,Completions generated successfully. Ready for display in your IDE.,8cedf7cb-5d25-40d8-8de3-f74ed47b272f,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = lower; i <= upper; i++) { -",214,1.0 +",214,1.0 2025-09-22T13:57:41.795569,[1],"//Create a function which returns the largest index of an element which // is not greater than or equal to the element immediately preceding it. If // no such element exists then return -1. The given array will not contain @@ -12048,7 +12048,7 @@ function can_arrange(arr){ } ",97.18820000125561,,Completions generated successfully. Ready for display in your IDE.,f07ef9db-46c4-43f5-8a36-6d96c2fd6610,1,deepseek-ai/deepseek-coder-1.3b-base," continue; -",74,1.0 +",74,1.0 2025-09-22T13:57:42.225916,[1],"from typing import List def sort_array(array: List[int]) -> List[int]: @@ -12075,7 +12075,7 @@ def sort_array(array: List[int]) -> List[int]: array.sort() else: array.sort(reverse=True) -",403,0.9657 +",403,0.9657 2025-09-22T13:57:42.416951,[1],"//For a given array of input numbers, calculate Mean Absolute Deviation // around the mean of this dataset. // Mean Absolute Deviation is the average absolute difference between each @@ -12095,7 +12095,7 @@ function mean_absolute_deviation(numbers){ }","} ",191.20030000340194,,Completions generated successfully. Ready for display in your IDE.,5c470597-966d-4d9c-999e-5eab81c2570d,1,deepseek-ai/deepseek-coder-1.3b-base," return sum_abs_diff / numbers.length; -",166,0.9809 +",166,0.9809 2025-09-22T13:57:42.871879,[1],"from typing import List def minSubArraySum(nums: List[int]) -> int: @@ -12112,7 +12112,7 @@ def minSubArraySum(nums: List[int]) -> int: ",454.7935000009602,,Completions generated successfully. Ready for display in your IDE.,dc827082-3b6f-49d7-9924-784a861699cb,1,deepseek-ai/deepseek-coder-1.3b-base," if nums[i] < nums[i - 1]: nums[i] = nums[i - 1] + nums[i] -",431,0.9923 +",431,0.9923 2025-09-22T13:57:43.042832,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -12167,7 +12167,7 @@ function by_length(arr){ return arr3; }",171.08600000210572,,Completions generated successfully. Ready for display in your IDE.,6bbd00d7-9948-4991-a560-72e39c8a9ebe,1,deepseek-ai/deepseek-coder-1.3b-base," arr3.push(""Eight""); -",144,1.0 +",144,1.0 2025-09-22T13:57:43.121738,[1],"//Write a function count_nums which takes an array of integers and returns // the number of elements which has a sum of digits > 0. // If a number is negative, then its first signed digit will be negative: @@ -12201,7 +12201,7 @@ function count_nums(arr){ }"," return count; }",78.18570000381442,,Completions generated successfully. Ready for display in your IDE.,37c2b707-51d8-4340-b1d9-12f542173105,1,deepseek-ai/deepseek-coder-1.3b-base," } -",54,1.0 +",54,1.0 2025-09-22T13:57:43.228634,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -12268,7 +12268,7 @@ function numerical_letter_grade(grades){ return grades_array; }",106.74990000552498,,Completions generated successfully. Ready for display in your IDE.,9418768c-11c9-49b4-8eda-ed6ecde0cbd2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",78,1.0 +",78,1.0 2025-09-22T13:57:43.318505,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12293,7 +12293,7 @@ class Problem { public static boolean sameChars(String s0, String s1) { return s0.chars().distinct().sorted().boxed().collect(Collectors.toList()).equals(s1.chars().distinct().sorted().boxed().collect(Collectors.toList()));",},89.36040000116918,,Completions generated successfully. Ready for display in your IDE.,76128692-364e-4308-905a-4702efc24dde,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:43.405146,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -12331,7 +12331,7 @@ function is_prime(n){ } ",86.8553000036627,,Completions generated successfully. Ready for display in your IDE.,dcfe468e-7bf8-4029-b617-79239e9575d2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:57:43.572973,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12371,7 +12371,7 @@ class Problem { } }",166.85599999618717,,Completions generated successfully. Ready for display in your IDE.,5cc1e1c5-381a-4953-aa17-ace3d7e93ae0,1,deepseek-ai/deepseek-coder-1.3b-base," for (String word: words) { -",140,0.9872 +",140,0.9872 2025-09-22T13:57:43.828199,[1],"//Filter an input array of strings only for ones that contain given substring // >>> filter_by_substring([], ""a"") // [] @@ -12385,7 +12385,7 @@ function filter_by_substring(strings, substring){ return filtered; }",255.46819999726722,,Completions generated successfully. Ready for display in your IDE.,db52aba6-f8af-4049-a7ab-2679a5114b1e,1,deepseek-ai/deepseek-coder-1.3b-base," if (strings[i].indexOf(substring) !== -1 ){ -",233,0.9656 +",233,0.9656 2025-09-22T13:57:43.939621,[1],"//Given the lengths of the three sides of a triangle. Return true if the three // sides form a right-angled triangle, false otherwise. // A right-angled triangle is a triangle in which one angle is right angle or @@ -12402,7 +12402,7 @@ function right_angle_triangle(a, b, c){ } ",110.98589999892283,,Completions generated successfully. Ready for display in your IDE.,92762183-1431-4a5f-9e77-d1e444f789ba,1,deepseek-ai/deepseek-coder-1.3b-base," } else{ -",84,1.0 +",84,1.0 2025-09-22T13:57:44.039953,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12443,7 +12443,7 @@ class Problem { } return res;",},100.25350000069011,,Completions generated successfully. Ready for display in your IDE.,07d354ab-84a9-4f13-9cf5-c360eb05f42d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",72,1.0 +",72,1.0 2025-09-22T13:57:44.262025,[1],"//Given an array of non-negative integers, return a cojs of the given array after sorting, // you will sort the given array in ascending order if the sum( first index value, last index value) is odd, // or sort it in descending order if the sum( first index value, last index value) is even. @@ -12468,7 +12468,7 @@ function sort_array(array){ }",221.56649999669753,,Completions generated successfully. Ready for display in your IDE.,04949976-02d3-4cb8-92a3-fe27c4b2ccea,1,deepseek-ai/deepseek-coder-1.3b-base," //if (sum % 2 === 0 ) { - ",197,0.8935 + ",197,0.8935 2025-09-22T13:57:44.408726,[1],"//Given a positive integer n, return a sorted array that has the odd numbers in collatz sequence. // The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined // as follows: start with any positive integer n. Then each term is obtained from the @@ -12512,7 +12512,7 @@ function get_odd_collatz(n){ ",146.12109999870881,,Completions generated successfully. Ready for display in your IDE.,1ed34b96-f82f-4ef4-af02-cc27174aea8b,1,deepseek-ai/deepseek-coder-1.3b-base," while (true){ -",121,0.9873 +",121,0.9873 2025-09-22T13:57:44.563800,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12549,7 +12549,7 @@ class Problem { } }",155.78930000629043,,Completions generated successfully. Ready for display in your IDE.,c991e16d-9680-4569-997d-37c7d8f4d218,1,deepseek-ai/deepseek-coder-1.3b-base," temp.append(c); -",132,1.0 +",132,1.0 2025-09-22T13:57:44.702757,[1],"//Input is a space-delimited string of numberals from 'zero' to 'nine'. // Valid choices are 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight' and 'nine'. // Return the string with numbers sorted from smallest to largest @@ -12574,7 +12574,7 @@ function sort_numbers(numbers){ return numString; }",137.92580000153976,,Completions generated successfully. Ready for display in your IDE.,78fb89e8-5a6a-4927-803f-808a75caf906,1,deepseek-ai/deepseek-coder-1.3b-base," ""eight"": 8, -",115,1.0 +",115,1.0 2025-09-22T13:57:44.769114,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12597,7 +12597,7 @@ class Problem { return result; } }",66.68109999736771,,Completions generated successfully. Ready for display in your IDE.,22575594-d402-4f13-8ae7-e926ef346c47,1,deepseek-ai/deepseek-coder-1.3b-base," - ",42,0.9407 + ",42,0.9407 2025-09-22T13:57:44.876949,[1],"//Given a positive integer n, return a sorted array that has the odd numbers in collatz sequence. // The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined // as follows: start with any positive integer n. Then each term is obtained from the @@ -12640,7 +12640,7 @@ function get_odd_collatz(n){ } ",107.313400003477,,Completions generated successfully. Ready for display in your IDE.,5b857e86-6b09-4380-be3c-793094cb3336,1,deepseek-ai/deepseek-coder-1.3b-base," else { -",82,1.0 +",82,1.0 2025-09-22T13:57:44.956725,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -12667,7 +12667,7 @@ function next_smallest(lst){ return second_smallest; }",79.17820000147913,,Completions generated successfully. Ready for display in your IDE.,34da66e3-3d1e-45c2-b2ed-299bdcaa78f0,1,deepseek-ai/deepseek-coder-1.3b-base," } -",58,1.0 +",58,1.0 2025-09-22T13:57:45.303707,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -12736,7 +12736,7 @@ function do_algebra(operator, operand){ ",346.9537000055425,,Completions generated successfully. Ready for display in your IDE.,a4fc3169-428e-4b26-b959-c9b30831ae71,1,deepseek-ai/deepseek-coder-1.3b-base," if(!do_again) {return answer;} else{return do_algebra(operator, operand)}; -",320,0.9883 +",320,0.9883 2025-09-22T13:57:45.411959,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12769,7 +12769,7 @@ class Problem { } }",108.53489999863086,,Completions generated successfully. Ready for display in your IDE.,3acaff7f-e14c-45ba-b496-2c802d47a4c3,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",84,1.0 +",84,1.0 2025-09-22T13:57:45.475644,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12808,7 +12808,7 @@ class Problem { }"," return b; } }",63.00409999676049,,Completions generated successfully. Ready for display in your IDE.,2c51da2b-298e-4837-82e1-d521e9c83dcc,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:57:45.654760,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -12847,7 +12847,7 @@ function valid_date(date){ } ",179.32830000063404,,Completions generated successfully. Ready for display in your IDE.,faec5c26-c06a-4d34-9794-d69389524834,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,0.9169 +",69,0.9169 2025-09-22T13:57:45.730288,[1],"//Write a function that takes an integer a and returns true // if this ingeger is a cube of some integer number. // Note: you may assume the input is always valid. @@ -12870,7 +12870,7 @@ function iscube(a){ }else{ return false;",},74.7655000013765,,Completions generated successfully. Ready for display in your IDE.,eeeca37f-087d-4d4f-8936-0bec8facb406,1,deepseek-ai/deepseek-coder-1.3b-base," } -",52,1.0 +",52,1.0 2025-09-22T13:57:46.426874,[1],"//You are given 2 words. You need to return true if the second word or any of its rotations is a substring in the first word // >>> cycpattern_check(""abcd"", ""abd"") // false @@ -12893,7 +12893,7 @@ function cycpattern_check(a, b){ ",696.9198000006145,,Completions generated successfully. Ready for display in your IDE.,70991dda-ac5c-487d-8891-380077cc79d3,1,deepseek-ai/deepseek-coder-1.3b-base," //console.log('b: '+i); console.log((new RegExp(`^${b}(?=${a})|\\Q${b}\\E(\\Q${b}\\E|${a}.*?)+$`)).test(a)); -",669,0.8914 +",669,0.8914 2025-09-22T13:57:46.678118,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -12935,7 +12935,7 @@ class Problem { } }",250.5167999988771,,Completions generated successfully. Ready for display in your IDE.,4310919a-47d6-4ae8-8d52-d9dbf8fc2780,1,deepseek-ai/deepseek-coder-1.3b-base," if (! Character.isLetter(s.charAt(i))){ -",223,0.9489 +",223,0.9489 2025-09-22T13:57:46.720212,[1],"from typing import List def triples_sum_to_zero(l: List[int]) -> bool: @@ -12962,7 +12962,7 @@ def triples_sum_to_zero(l: List[int]) -> bool: if l[i] + l[j] + l[k] == 0: return True return False -",41.92530000000261,,Completions generated successfully. Ready for display in your IDE.,76c0b9cc-eb9f-4992-b32c-92dba1d83bb0,1,deepseek-ai/deepseek-coder-1.3b-base,,19,1.0 +",41.92530000000261,,Completions generated successfully. Ready for display in your IDE.,76c0b9cc-eb9f-4992-b32c-92dba1d83bb0,1,deepseek-ai/deepseek-coder-1.3b-base,,19,1.0 2025-09-22T13:57:46.824249,[1],"//Return array of all prefixes from shortest to longest of the input string // >>> all_prefixes(""abc"") // [""a"", ""ab"", ""abc""] @@ -12973,7 +12973,7 @@ function all_prefixes(string){ }","} ",103.99909999978263,,Completions generated successfully. Ready for display in your IDE.,56905171-f84b-44c8-bbb0-f21c2586db3c,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",82,1.0 +",82,1.0 2025-09-22T13:57:46.918190,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -13002,7 +13002,7 @@ function f(n){ return result; }",94.01859999343287,,Completions generated successfully. Ready for display in your IDE.,273f4e8c-0494-438f-8ae3-94834c93e5e7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:47.026056,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -13027,7 +13027,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: return groups ",107.04709999845363,,Completions generated successfully. Ready for display in your IDE.,3b28ebb0-0339-4e9c-a0cb-7739f6176e8e,1,deepseek-ai/deepseek-coder-1.3b-base," group = '' -",83,1.0 +",83,1.0 2025-09-22T13:57:47.211475,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13064,7 +13064,7 @@ class Problem { } }",185.25640000007115,,Completions generated successfully. Ready for display in your IDE.,d6fe6a86-52d5-4d7e-a582-d9698b2fe85b,1,deepseek-ai/deepseek-coder-1.3b-base," if (open_paren == 0) { -",163,1.0 +",163,1.0 2025-09-22T13:57:47.306241,[1],"//We have an array 'arr' of N integers arr[1], arr[2], ..., arr[N].The // numbers in the array will be randomly ordered. Your task is to determine if // it is possible to get an array sorted in non-decreasing order by performing @@ -13101,7 +13101,7 @@ function move_one_ball(arr){ } }",94.72920000553131,,Completions generated successfully. Ready for display in your IDE.,40a06913-6af0-46d8-84d1-af9ba6ff90b8,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:57:47.569502,[1],"//Given an array of strings, where each string consists of only digits, return an array. // Each element i of the output should be ""the number of odd elements in the // string i of the input."" where all the i's should be replaced by the number @@ -13124,7 +13124,7 @@ function odd_count(lst){ return new_lst; }",263.01059999968857,,Completions generated successfully. Ready for display in your IDE.,4ab11bd1-125a-48c6-be2e-266ae6328252,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i = 0; i < lst.length; i++) { -",238,0.9962 +",238,0.9962 2025-09-22T13:57:47.798056,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13164,7 +13164,7 @@ class Problem { } }",228.28669999580598,,Completions generated successfully. Ready for display in your IDE.,afe7df12-c7e3-4c23-a1b0-859920342f87,1,deepseek-ai/deepseek-coder-1.3b-base," if (isPrime == true && i != 1) { -",203,0.9587 +",203,0.9587 2025-09-22T13:57:48.022326,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -13220,7 +13220,7 @@ function by_length(arr){ return arr3; }",224.68769999977667,,Completions generated successfully. Ready for display in your IDE.,eb7a4383-8afb-4007-b420-8a00eaf30699,1,deepseek-ai/deepseek-coder-1.3b-base," // Your code here... var arr2 = new Array(); -",193,0.929 +",193,0.929 2025-09-22T13:57:48.128292,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -13287,7 +13287,7 @@ function numerical_letter_grade(grades){ return grades_array; }",105.73129999829689,,Completions generated successfully. Ready for display in your IDE.,0ed43ab1-8ed8-4a86-b252-b890cf21207a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",77,1.0 +",77,1.0 2025-09-22T13:57:48.236162,[1],"from typing import List def add_elements(arr: List[int], k: int) -> int: @@ -13309,7 +13309,7 @@ def add_elements(arr: List[int], k: int) -> int: if num < 100:"," return total ",107.33970000001136,,Completions generated successfully. Ready for display in your IDE.,3717eab6-2be5-4e98-a124-3e0afbd4c3cb,1,deepseek-ai/deepseek-coder-1.3b-base," total += num -",84,1.0 +",84,1.0 2025-09-22T13:57:48.414096,[1],"//Given an array of numbers, return the sum of squares of the numbers // in the array that are odd. Ignore numbers that are negative or not integers. // >>> double_the_difference([1, 3, 2, 0]) @@ -13332,7 +13332,7 @@ function double_the_difference(lst){ continue; } } -",155,1.0 +",155,1.0 2025-09-22T13:57:48.567959,[1],"//Write a function that takes a string and returns an ordered version of it. // Ordered version of string, is a string where all words (separated by space) // are replaced by a new word where all the characters arranged in @@ -13354,7 +13354,7 @@ function anti_shuffle(s){ return result.join("" ""); }",153.84079999785172,,Completions generated successfully. Ready for display in your IDE.,767112fd-1aab-43f1-afaa-44ba70aabed2,1,deepseek-ai/deepseek-coder-1.3b-base," strings.forEach((str) => { -",131,0.9949 +",131,0.9949 2025-09-22T13:57:48.682111,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13392,7 +13392,7 @@ class Problem { } }",114.51379999925848,,Completions generated successfully. Ready for display in your IDE.,69ede347-2f7c-4679-afff-059865201795,1,deepseek-ai/deepseek-coder-1.3b-base," min++; -",89,1.0 +",89,1.0 2025-09-22T13:57:48.967247,[1],"from typing import List def rescale_to_unit(numbers: List[float]) -> List[float]: @@ -13405,7 +13405,7 @@ def rescale_to_unit(numbers: List[float]) -> List[float]: if _min == _max: return [0.5] * len(numbers)",,284.91809999832185,,Completions generated successfully. Ready for display in your IDE.,e107b80f-90b4-40dc-ab6c-ec165c0db2df,1,deepseek-ai/deepseek-coder-1.3b-base," else: - return [_value / (_max - _min) for _value in numbers]",262,0.9557 + return [_value / (_max - _min) for _value in numbers]",262,0.9557 2025-09-22T13:57:49.010723,[1],"//Insert a number 'delimeter' between every two consecutive elements of input array `numbers' // >>> intersperse([], 4) // [] @@ -13419,7 +13419,7 @@ function intersperse(numbers, delimeter){ result.push(delimeter); } } - return result;",,42.96790000080364,,Completions generated successfully. Ready for display in your IDE.,91a71f47-2650-4dd5-aa56-817cc866490c,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 + return result;",,42.96790000080364,,Completions generated successfully. Ready for display in your IDE.,91a71f47-2650-4dd5-aa56-817cc866490c,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:57:49.136894,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13459,7 +13459,7 @@ class Problem { }"," } }",126.19529999938095,,Completions generated successfully. Ready for display in your IDE.,e5b447fc-f4e1-42ce-b657-e765f394b868,1,deepseek-ai/deepseek-coder-1.3b-base," return arrayList; -",100,1.0 +",100,1.0 2025-09-22T13:57:49.437945,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13489,7 +13489,7 @@ class Problem { } }",300.56770000373945,,Completions generated successfully. Ready for display in your IDE.,0226d0cb-02f7-43ad-9d24-c667f7e69b80,1,deepseek-ai/deepseek-coder-1.3b-base," if (!vowels.contains(text.substring(i, i + 1))) { -",278,0.9814 +",278,0.9814 2025-09-22T13:57:49.534973,[1],"//This function takes two positive numbers x and y and returns the // biggest even integer number that is in the range [x, y] inclusive. If // there's no such number, then the function should return -1. @@ -13507,7 +13507,7 @@ function choose_num(x, y){ }","} ",96.73479999764822,,Completions generated successfully. Ready for display in your IDE.,bcad8828-8500-44d8-b39a-e81810c1dbbb,1,deepseek-ai/deepseek-coder-1.3b-base," return num; -",72,1.0 +",72,1.0 2025-09-22T13:57:49.623862,[1],"def vowels_count(s: str) -> int: """"""Write a function vowels_count which takes a string representing a word as input and returns the number of vowels in the string. @@ -13529,7 +13529,7 @@ function choose_num(x, y){ if (s[i] == 'y' and i == len(s) - 1): count += 1 i += 1",,88.99490000476362,,Completions generated successfully. Ready for display in your IDE.,695d1cf4-8f49-4b47-b9a2-20f76d02c1df,1,deepseek-ai/deepseek-coder-1.3b-base," - return count",63,1.0 + return count",63,1.0 2025-09-22T13:57:49.886484,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -13553,7 +13553,7 @@ function vowels_count(s){ } ",262.32019999588374,,Completions generated successfully. Ready for display in your IDE.,68430855-e7e5-4f01-add1-7e9a1ebbbd00,1,deepseek-ai/deepseek-coder-1.3b-base," if (vowels.indexOf(s[i]) !== -1) { -",237,0.9545 +",237,0.9545 2025-09-22T13:57:50.008084,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13587,7 +13587,7 @@ class Problem { } }",121.5161999934935,,Completions generated successfully. Ready for display in your IDE.,803b8347-82e3-4432-87cb-75da2865a135,1,deepseek-ai/deepseek-coder-1.3b-base," fib += prev; -",96,1.0 +",96,1.0 2025-09-22T13:57:50.068978,[1],"//You are given two intervals, // where each interval is a pair of integers. For example, interval = (start, end) = (1, 2). // The given intervals are closed which means that the interval (start, end) @@ -13616,7 +13616,7 @@ function intersection(interval1, interval2){ let length = end - start;"," return primes.includes(length) ? ""YES"" : ""NO""; } ",60.48449999798322,,Completions generated successfully. Ready for display in your IDE.,ed6858dc-9aa9-4240-ad3e-1a32e2acc7eb,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:57:50.528179,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13636,7 +13636,7 @@ class Problem { float mean = (float)numbers.stream().mapToDouble(Float::floatValue).average().getAsDouble();"," } }",459.2399999964982,,Completions generated successfully. Ready for display in your IDE.,b65d5385-b51a-49d9-939a-6a79c008389d,1,deepseek-ai/deepseek-coder-1.3b-base," return (float)(Math.abs(numbers.stream().mapToDouble(x -> Math.abs(x-mean)).average().getAsDouble())); -",411,0.9852 +",411,0.9852 2025-09-22T13:57:50.819456,[1],"from typing import List def sort_third(l: List[int]) -> List[int]: @@ -13654,7 +13654,7 @@ def sort_third(l: List[int]) -> List[int]: return l ",290.7844000001205,,Completions generated successfully. Ready for display in your IDE.,5d07ee89-4c39-4758-a503-63de8f772668,1,deepseek-ai/deepseek-coder-1.3b-base," l[j], l[j + 3] = l[j + 3], l[j] -",267,1.0 +",267,1.0 2025-09-22T13:57:50.935379,[1],"from typing import List def rolling_max(numbers: List[int]) -> List[int]: @@ -13668,7 +13668,7 @@ def rolling_max(numbers: List[int]) -> List[int]: return [max(numbers[:i + 1]) for i in range(len(numbers))] ",115.36730000079842,,Completions generated successfully. Ready for display in your IDE.,cbdee79b-ab9c-43da-9a95-7357b5a86c88,1,deepseek-ai/deepseek-coder-1.3b-base," if not numbers: -",83,0.9286 +",83,0.9286 2025-09-22T13:57:51.023791,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13697,7 +13697,7 @@ class Problem { } }",88.64419999736128,,Completions generated successfully. Ready for display in your IDE.,6a89f875-1f41-4b91-892f-01d7c8ad8633,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:57:51.201416,[1],"//Given a positive integer N, return the total sum of its digits in binary. // Example // >>> solve(1000) @@ -13720,7 +13720,7 @@ function solve(N){ return total.toString(2); }",177.0565999977407,,Completions generated successfully. Ready for display in your IDE.,27a75b57-f22f-41b6-8943-a547a7b6260c,1,deepseek-ai/deepseek-coder-1.3b-base," const numToString = (num).toString(); -",155,0.8308 +",155,0.8308 2025-09-22T13:57:51.397783,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -13792,7 +13792,7 @@ function minPath(grid, k){ return path; }",196.45470000250498,,Completions generated successfully. Ready for display in your IDE.,c1abf618-4f22-4801-a46f-72ae8db93d5a,1,deepseek-ai/deepseek-coder-1.3b-base," if(x < n - 1){ -",166,1.0 +",166,1.0 2025-09-22T13:57:52.100695,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13824,7 +13824,7 @@ class Problem { throw new IllegalArgumentException(""Invalid input""); } else if (!(k > 0)) { - for (int",677,0.8953 + for (int",677,0.8953 2025-09-22T13:57:52.165428,[1],"//Given the lengths of the three sides of a triangle. Return true if the three // sides form a right-angled triangle, false otherwise. // A right-angled triangle is a triangle in which one angle is right angle or @@ -13840,7 +13840,7 @@ function right_angle_triangle(a, b, c){ } else { return false; }",,63.92059999780031,,Completions generated successfully. Ready for display in your IDE.,49d11ac7-e096-4b32-9f1e-7810746013ae,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:57:52.269807,[1],"from typing import List def all_prefixes(string: str) -> List[str]: @@ -13852,7 +13852,7 @@ def all_prefixes(string: str) -> List[str]: prefixes.append(string[:i+1]) return prefixes ",104.45189999882132,,Completions generated successfully. Ready for display in your IDE.,7bdbe105-ef84-4cca-84e0-37c071f31775,1,deepseek-ai/deepseek-coder-1.3b-base," prefixes = [] -",81,1.0 +",81,1.0 2025-09-22T13:57:52.447371,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -13920,7 +13920,7 @@ class Problem { } }",177.6398000001791,,Completions generated successfully. Ready for display in your IDE.,b8401951-ebcd-490a-8568-75bbbd9e4ccd,1,deepseek-ai/deepseek-coder-1.3b-base," long running_max = 0; -",146,1.0 +",146,1.0 2025-09-22T13:57:52.500409,[1],"from typing import List def triples_sum_to_zero(l: List[int]) -> bool: @@ -13947,7 +13947,7 @@ def triples_sum_to_zero(l: List[int]) -> bool: return True return False ",52.79129999689758,,Completions generated successfully. Ready for display in your IDE.,c7f4fd1c-ee17-4188-aab9-6b817bb096af,1,deepseek-ai/deepseek-coder-1.3b-base," -",30,1.0 +",30,1.0 2025-09-22T13:57:52.541524,[1],"from typing import List def sort_array(array: List[int]) -> List[int]: @@ -13970,7 +13970,7 @@ def sort_array(array: List[int]) -> List[int]: [6, 5, 4, 3, 2, 1, 0] """""" "," return sorted(array, reverse=True) if (array[0] + array[-1]) % 2 == 0 else sorted(array) - return array",40.960599995742086,,Completions generated successfully. Ready for display in your IDE.,d2e42fd0-a1a7-4e12-a030-cee097c7b759,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 + return array",40.960599995742086,,Completions generated successfully. Ready for display in your IDE.,d2e42fd0-a1a7-4e12-a030-cee097c7b759,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 2025-09-22T13:57:53.256359,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -14043,7 +14043,7 @@ function do_algebra(operator, operand){ do_again = true; break; case '/': - operand[count] = operand[count] /",687,0.9801 + operand[count] = operand[count] /",687,0.9801 2025-09-22T13:57:53.310919,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14072,7 +14072,7 @@ class Problem { return newString.toString(); } }",53.89229999855161,,Completions generated successfully. Ready for display in your IDE.,daf74761-e8e9-4f46-969e-e947ba060d53,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,0.9821 +",31,0.9821 2025-09-22T13:57:53.528305,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14112,7 +14112,7 @@ class Problem { } }",217.32340000016848,,Completions generated successfully. Ready for display in your IDE.,1dddda31-80fd-4fd3-9734-483afcccf323,1,deepseek-ai/deepseek-coder-1.3b-base," Set uniqueCharacters = new HashSet<>(); -",192,1.0 +",192,1.0 2025-09-22T13:57:53.625183,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14156,7 +14156,7 @@ class Problem { } }",96.07279999909224,,Completions generated successfully. Ready for display in your IDE.,d59ef2af-890d-4a90-90c6-f83b9b677aa6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:57:53.706437,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14223,7 +14223,7 @@ class Problem { } } }",81.68919999297941,,Completions generated successfully. Ready for display in your IDE.,05495bd2-d087-4042-bfb0-75cf84e8eaa0,1,deepseek-ai/deepseek-coder-1.3b-base," -",52,1.0 +",52,1.0 2025-09-22T13:57:53.981005,[1],"from typing import List def sort_array(arr: List[int]) -> List[int]: @@ -14240,7 +14240,7 @@ def sort_array(arr: List[int]) -> List[int]: >>> sort_array([1, 0, 2, 3, 4]) [0, 1, 2, 3, 4] """""" -",,273.7691000002087,,Completions generated successfully. Ready for display in your IDE.,2777b5e0-9b50-4226-b523-2e59b3cc472d,1,deepseek-ai/deepseek-coder-1.3b-base," return sorted(arr, key=lambda x: (bin(x).count('1'), x))",251,0.9968 +",,273.7691000002087,,Completions generated successfully. Ready for display in your IDE.,2777b5e0-9b50-4226-b523-2e59b3cc472d,1,deepseek-ai/deepseek-coder-1.3b-base," return sorted(arr, key=lambda x: (bin(x).count('1'), x))",251,0.9968 2025-09-22T13:57:54.158807,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14288,7 +14288,7 @@ class Problem { } }",178.18830000032904,,Completions generated successfully. Ready for display in your IDE.,67817959-8005-47f5-987d-182828a91113,1,deepseek-ai/deepseek-coder-1.3b-base," indexMin = indexPlanet2; -",149,1.0 +",149,1.0 2025-09-22T13:57:54.347476,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14327,7 +14327,7 @@ class Problem { } }",188.20300000516,,Completions generated successfully. Ready for display in your IDE.,d4a4f1a4-69d1-46cd-b845-7ebfae0b8275,1,deepseek-ai/deepseek-coder-1.3b-base," if (currSum < minSum) { -",162,1.0 +",162,1.0 2025-09-22T13:57:54.459384,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14374,7 +14374,7 @@ class Problem { } }",111.82900000130758,,Completions generated successfully. Ready for display in your IDE.,e0488f7c-c41c-45a5-a4a7-24bc317dea6a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:57:54.646888,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14412,7 +14412,7 @@ class Problem { }",186.74069999542553,,Completions generated successfully. Ready for display in your IDE.,7338edaf-6a3e-474e-9586-b18ccb2172a4,1,deepseek-ai/deepseek-coder-1.3b-base," temp = new StringBuffer(); } -",162,0.9957 +",162,0.9957 2025-09-22T13:57:54.712015,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14435,7 +14435,7 @@ class Problem { } return greatestCommonDivisor(b, a % b); }",,65.39199999679113,,Completions generated successfully. Ready for display in your IDE.,baa19ba7-0de1-474c-b574-bc23214193fa,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:57:55.032333,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14483,7 +14483,7 @@ class Problem { } }",319.84790000569774,,Completions generated successfully. Ready for display in your IDE.,fe36cf7f-4bfa-41d0-8684-017e096f4fda,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = indexMin + 1; i <= indexMax - 1; ++i) { -",291,0.9946 +",291,0.9946 2025-09-22T13:57:55.191598,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14561,7 +14561,7 @@ class Problem { } }",159.1757999995025,,Completions generated successfully. Ready for display in your IDE.,bad9a35b-1336-4e9e-b7c1-7ab669863562,1,deepseek-ai/deepseek-coder-1.3b-base," num -= 900; -",133,1.0 +",133,1.0 2025-09-22T13:57:55.463797,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14592,7 +14592,7 @@ class Problem { while (x % i == 0) { x /= i; } -",247,1.0 +",247,1.0 2025-09-22T13:57:55.561797,[1],"//Given a string s and a natural number n, you have been tasked to implement // a function that returns an array of all words from string s that contain exactly // n consonants, in order these words appear in the string s. @@ -14618,7 +14618,7 @@ function select_words(s, n){ } ",97.99309999652905,,Completions generated successfully. Ready for display in your IDE.,713573c4-f90e-4421-a4a0-1c84a0395cac,1,deepseek-ai/deepseek-coder-1.3b-base," return []; -",73,1.0 +",73,1.0 2025-09-22T13:57:55.634951,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14658,7 +14658,7 @@ class Problem { } return arrayList; }",,72.3805999950855,,Completions generated successfully. Ready for display in your IDE.,25e5addd-d9ce-44dd-8f16-753a80d9e72b,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T13:57:56.207224,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14685,7 +14685,7 @@ class Problem { Float nextNum = numbers.get(i + 1); double diff = Math.abs(nextNum - num); if (diff <= threshold) { -",546,0.9678 +",546,0.9678 2025-09-22T13:57:56.317882,[1],"//Given a string of words, return an array of words split on whitespace, if no whitespaces exists in the text you // should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the // alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 @@ -14714,7 +14714,7 @@ function split_words(txt){ }"," } }",110.28660000010859,,Completions generated successfully. Ready for display in your IDE.,bd2be2dd-a323-4044-a84d-58b9b85a16f6,1,deepseek-ai/deepseek-coder-1.3b-base," return count; -",86,1.0 +",86,1.0 2025-09-22T13:57:56.455558,[1],"//Input to this function is a string representing musical notes in a special ASCII format. // Your task is to parse this string and return array of integers corresponding to how many beats does each // not last. @@ -14738,7 +14738,7 @@ function parse_music(music_string){ } ",137.0803999961936,,Completions generated successfully. Ready for display in your IDE.,32811252-6413-4b86-b192-c3c1faeec0b6,1,deepseek-ai/deepseek-coder-1.3b-base," song.push(1); -",114,1.0 +",114,1.0 2025-09-22T13:57:56.548141,[1],"def largest_prime_factor(n: int) -> int: """"""Return the largest prime factor of n. Assume n > 1 and is not a prime. >>> largest_prime_factor(13195) @@ -14756,7 +14756,7 @@ function parse_music(music_string){ factor += 1"," ",92.722400004277,,Completions generated successfully. Ready for display in your IDE.,8f0b3c6c-5ea6-461b-a437-e60715466752,1,deepseek-ai/deepseek-coder-1.3b-base," return largest -",72,0.9431 +",72,0.9431 2025-09-22T13:57:56.640063,[1],"//For a given array of integers, return an array consisting of a sum and a product of all the integers in an array. // Empty sum should be equal to 0 and empty product should be equal to 1. // >>> sum_product([]) @@ -14775,7 +14775,7 @@ function sum_product(numbers){ } ",91.81270000408404,,Completions generated successfully. Ready for display in your IDE.,8f1fa288-6426-4ee4-8baf-ecc8766370af,1,deepseek-ai/deepseek-coder-1.3b-base," return []; -",70,0.9003 +",70,0.9003 2025-09-22T13:57:56.713560,[1],"//In this problem, you will implement a function that takes two arrays of numbers, // and determines whether it is possible to perform an exchange of elements // between them to make lst1 an array of only even numbers. @@ -14813,7 +14813,7 @@ function exchange(lst1, lst2){ } } ",73.538999997254,,Completions generated successfully. Ready for display in your IDE.,86fdd628-0492-4b76-9244-025d09c5f26e,1,deepseek-ai/deepseek-coder-1.3b-base," - ",49,1.0 + ",49,1.0 2025-09-22T13:57:56.824643,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -14855,7 +14855,7 @@ function compare_one(a, b){ } ",110.28090000036173,,Completions generated successfully. Ready for display in your IDE.,92cc05b0-b26d-4a74-8c44-8d28d98b2d75,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",86,1.0 +",86,1.0 2025-09-22T13:57:56.941170,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14923,7 +14923,7 @@ class Problem { } }",116.88620000495575,,Completions generated successfully. Ready for display in your IDE.,663c516d-bfdd-47c1-9632-3e787ec12e3c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",84,1.0 +",84,1.0 2025-09-22T13:57:57.056599,[1],"//Given an array of integers nums, find the minimum sum of any non-empty sub-array // of nums. // Example @@ -14945,7 +14945,7 @@ function minSubArraySum(nums){ } ",114.85130000073696,,Completions generated successfully. Ready for display in your IDE.,5accda74-f719-448e-b80d-6308d842ecea,1,deepseek-ai/deepseek-coder-1.3b-base," sum = 0; -",93,1.0 +",93,1.0 2025-09-22T13:57:57.157285,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -14989,7 +14989,7 @@ class Problem { } }",100.24189999967348,,Completions generated successfully. Ready for display in your IDE.,2eb24595-c233-482f-8226-0535121f9183,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T13:57:57.350729,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15042,7 +15042,7 @@ class Problem { } }",192.92110000242246,,Completions generated successfully. Ready for display in your IDE.,b7e02d22-d6b3-4e69-ad79-3a33c9500391,1,deepseek-ai/deepseek-coder-1.3b-base," if (minIndex == -1) { -",162,1.0 +",162,1.0 2025-09-22T13:57:57.480221,[1],"//You're a hungry rabbit, and you already have eaten a certain number of carrots, // but now you need to eat more carrots to complete the day's meals. // you should return an array of [ total number of eaten carrots after your meals, @@ -15073,7 +15073,7 @@ function eat(number, need, remaining){ "," else return [number + need, remaining - need]; } ",130.083600000944,,Completions generated successfully. Ready for display in your IDE.,230fb330-c2e9-4f2f-bcdb-5aacbb00b55e,1,deepseek-ai/deepseek-coder-1.3b-base," //your code here... - ",105,0.8994 + ",105,0.8994 2025-09-22T13:57:57.604748,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -15108,7 +15108,7 @@ function eat(number, need, remaining){ if e_name not in ['txt', 'exe', 'dll']: return 'No'"," ",123.99530000402592,,Completions generated successfully. Ready for display in your IDE.,2c3a4b0c-62af-4a65-81a2-5c6a3568fb96,1,deepseek-ai/deepseek-coder-1.3b-base," - return 'Yes'",99,0.9556 + return 'Yes'",99,0.9556 2025-09-22T13:57:57.830431,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -15137,7 +15137,7 @@ function f(n){ return result; }",225.43460000451887,,Completions generated successfully. Ready for display in your IDE.,c2a5da7e-106c-46ab-a975-df27758bac29,1,deepseek-ai/deepseek-coder-1.3b-base," for(var k = 1; k <= i; k++){ -",199,1.0 +",199,1.0 2025-09-22T13:57:57.952905,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15181,7 +15181,7 @@ class Problem { }).count(); } }",122.04880000354024,,Completions generated successfully. Ready for display in your IDE.,b840730b-55cd-401f-a077-e81fa8e6f964,1,deepseek-ai/deepseek-coder-1.3b-base," class helper { -",79,0.994 +",79,0.994 2025-09-22T13:57:58.156381,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15213,7 +15213,7 @@ class Problem { } }",203.02999999694293,,Completions generated successfully. Ready for display in your IDE.,58462354-9588-4ddf-a1b5-f5c18e173fd7,1,deepseek-ai/deepseek-coder-1.3b-base," for (String paren:paren_strings) { -",177,1.0 +",177,1.0 2025-09-22T13:57:58.423628,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15247,7 +15247,7 @@ class Problem { } }",267.48879999649944,,Completions generated successfully. Ready for display in your IDE.,0c21915c-4491-4f99-96a1-db6e221c1633,1,deepseek-ai/deepseek-coder-1.3b-base," if ((fib & 1) == 0 && isPrime(fib)) { -",242,0.9618 +",242,0.9618 2025-09-22T13:57:58.518957,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -15302,7 +15302,7 @@ function by_length(arr){ return arr3; }",95.15790000295965,,Completions generated successfully. Ready for display in your IDE.,3ec7e928-3714-423d-82cd-f4f1baf818b2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:57:58.608792,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15332,7 +15332,7 @@ class Problem { } return newString.toString();",},89.62469999823952,,Completions generated successfully. Ready for display in your IDE.,344eed9e-544e-4f22-8674-1b18e7ec7356,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:57:59.244877,[1],"//Given a positive integer n, return a sorted array that has the odd numbers in collatz sequence. // The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined // as follows: start with any positive integer n. Then each term is obtained from the @@ -15378,7 +15378,7 @@ function get_odd_collatz(n){ // console.log('Odd Numbers', odd_numbers); if (collatz_sequence[i] === 1 ) { -",608,0.9441 +",608,0.9441 2025-09-22T13:57:59.329512,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -15416,7 +15416,7 @@ function find_max(words){ } return max_unique_word; }",84.69109999714419,,Completions generated successfully. Ready for display in your IDE.,fd328ccc-7139-476c-a04a-fd5ac30b139c,1,deepseek-ai/deepseek-coder-1.3b-base,"else { -",59,0.9714 +",59,0.9714 2025-09-22T13:57:59.393453,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -15449,7 +15449,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",63.64929999836022,,Completions generated successfully. Ready for display in your IDE.,65abea14-38d6-4e8d-8f20-b21e4286def6,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:57:59.436515,[1],"from typing import List def below_zero(operations: List[int]) -> bool: @@ -15467,7 +15467,7 @@ def below_zero(operations: List[int]) -> bool: if balance < 0: return True"," print(below_zero([1, 2, 3])) -print(below_zero([1, 2, -4, 5]))",42.78799999883631,,Completions generated successfully. Ready for display in your IDE.,c9128362-ab47-4c82-8c7e-2afa322be4a9,1,deepseek-ai/deepseek-coder-1.3b-base,,21,0.4378 +print(below_zero([1, 2, -4, 5]))",42.78799999883631,,Completions generated successfully. Ready for display in your IDE.,c9128362-ab47-4c82-8c7e-2afa322be4a9,1,deepseek-ai/deepseek-coder-1.3b-base,,21,0.4378 2025-09-22T13:57:59.646985,[1],"//Given a positive floating point number, it can be decomposed into // and integer part (largest integer smaller than given number) and decimals // (leftover part always smaller than 1). @@ -15477,7 +15477,7 @@ print(below_zero([1, 2, -4, 5]))",42.78799999883631,,Completions generated succe function truncate_number(number){ ","} ",210.6597000019974,,Completions generated successfully. Ready for display in your IDE.,8b634f6f-2f91-437f-96dc-c486bf8d990c,1,deepseek-ai/deepseek-coder-1.3b-base," return Math.trunc(number); -",122,0.9595 +",122,0.9595 2025-09-22T13:57:59.792432,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15524,7 +15524,7 @@ class Problem { } }",144.91629999974975,,Completions generated successfully. Ready for display in your IDE.,f6167c7d-eb39-471a-8f0b-8b4c318d7027,1,deepseek-ai/deepseek-coder-1.3b-base," count += units; -",115,1.0 +",115,1.0 2025-09-22T13:57:59.921927,[1],"//This function takes an array l and returns an array l' such that // l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal // to the values of the corresponding indicies of l, but sorted. @@ -15554,7 +15554,7 @@ function sort_third(l){ return x; }",128.8979999953881,,Completions generated successfully. Ready for display in your IDE.,4ccf9eb3-bd0b-404f-86bd-87d61ef26ccc,1,deepseek-ai/deepseek-coder-1.3b-base," let k=0; -",103,1.0 +",103,1.0 2025-09-22T13:58:00.011389,[1],"//The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fibfib(0) == 0 // fibfib(1) == 0 @@ -15580,7 +15580,7 @@ function fibfib(n){ ",89.90990000165766,,Completions generated successfully. Ready for display in your IDE.,8163c0a2-b7e1-481f-8aab-d4c381c1a93c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",66,1.0 +",66,1.0 2025-09-22T13:58:00.110085,[1],"//Given an array of numbers, return whether or not they are sorted // in ascending order. If array has more than 1 duplicate of the same // number, return false. Assume no negative numbers and only integers. @@ -15622,7 +15622,7 @@ function is_sorted(lst){ } ",98.23740000138059,,Completions generated successfully. Ready for display in your IDE.,d5a4e719-aa76-49b5-996e-9c9547a72b79,1,deepseek-ai/deepseek-coder-1.3b-base," } -",73,1.0 +",73,1.0 2025-09-22T13:58:00.813493,[1],"//triples_sum_to_zero takes an array of integers as an input. // it returns true if there are three distinct elements in the array that // sum to zero, and false otherwise. @@ -15653,7 +15653,7 @@ function triples_sum_to_zero(l){ var a = [...new Set((Array).concat(l))]; console.log(""a"", a); - for (let x=0 ;x>> median([3, 1, 2, 4, 5]) // 3 @@ -15836,7 +15836,7 @@ function median(l){ } else { return sorted[middle]; }",,61.20370000280673,,Completions generated successfully. Ready for display in your IDE.,cdaa4a54-a3c1-455a-84f0-27c3d915079b,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:58:01.364529,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -15885,7 +15885,7 @@ function prime_fib(n){ } return fib(fibNum - 1); }",58.18900000303984,,Completions generated successfully. Ready for display in your IDE.,a7ee50b9-b8e2-4125-8a42-5f40afd34fc8,1,deepseek-ai/deepseek-coder-1.3b-base," -",36,1.0 +",36,1.0 2025-09-22T13:58:01.426050,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -15928,7 +15928,7 @@ function histogram(test){ return result; } ",61.01840000337688,,Completions generated successfully. Ready for display in your IDE.,9c419bc2-bab9-4268-b01c-4756be381a72,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:58:01.490021,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15959,7 +15959,7 @@ class Problem { return sum; } }",63.599900000554044,,Completions generated successfully. Ready for display in your IDE.,cd9bd909-030d-4b81-ac2b-03d0915b5265,1,deepseek-ai/deepseek-coder-1.3b-base," -",40,1.0 +",40,1.0 2025-09-22T13:58:01.657245,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -15981,7 +15981,7 @@ class Problem { } }",166.9605999995838,,Completions generated successfully. Ready for display in your IDE.,eeb26d75-4862-4b3a-a6b0-77cf87afd7f1,1,deepseek-ai/deepseek-coder-1.3b-base," sum += lst.get(i); -",145,1.0 +",145,1.0 2025-09-22T13:58:01.817003,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -16039,7 +16039,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",159.71469999931287,,Completions generated successfully. Ready for display in your IDE.,f4143e84-a898-4b52-92d2-d4de3e5e558e,1,deepseek-ai/deepseek-coder-1.3b-base," elif x > 2.3: -",131,1.0 +",131,1.0 2025-09-22T13:58:01.917162,[1],"//Write a function that accepts an array of strings as a parameter, // deletes the strings that have odd lengths from it, // and returns the resulted array with a sorted order, @@ -16083,7 +16083,7 @@ function sorted_list_sum(lst){ var a = [""aa"", ""a"", ""aaa""]; var b = [""ab"", ""a"", ""aaa"", ""cd""];",99.94749999896158,,Completions generated successfully. Ready for display in your IDE.,6d325a6b-73bf-4ad4-983b-0d00cbf68d8f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,0.8445 +",71,0.8445 2025-09-22T13:58:02.029301,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -16155,7 +16155,7 @@ function minPath(grid, k){ currCell = nextCell;"," return path; }",112.17999999644235,,Completions generated successfully. Ready for display in your IDE.,69e6657d-3813-446a-a613-8245d985473f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:58:02.176519,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -16189,7 +16189,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",146.77110000047833,,Completions generated successfully. Ready for display in your IDE.,2070a93a-72d8-421a-956d-99518559d7c2,1,deepseek-ai/deepseek-coder-1.3b-base," temp2_index = j -",122,1.0 +",122,1.0 2025-09-22T13:58:02.241578,[1],"//You are given a string s. // if s[i] is a letter, reverse its case from lower to upper or vise versa, // otherwise keep it as it is. @@ -16216,7 +16216,7 @@ function solve(s){ return swapped.join(''); } return s.split('').reverse().join('');",,64.83370000205468,,Completions generated successfully. Ready for display in your IDE.,f008d747-1dd3-436d-b202-579936b2b0fa,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:58:02.296913,[1],"def prime_length(string: str) -> bool: """"""Write a function that takes a string and returns True if the string length is a prime number or False otherwise @@ -16240,7 +16240,7 @@ function solve(s){ else: return False"," ",55.3622000006726,,Completions generated successfully. Ready for display in your IDE.,9b067314-ce87-408f-a067-159468f1bcbe,1,deepseek-ai/deepseek-coder-1.3b-base," -",23,1.0 +",23,1.0 2025-09-22T13:58:02.570047,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -16266,7 +16266,7 @@ function string_xor(a, b){ return result; }",272.8852000000188,,Completions generated successfully. Ready for display in your IDE.,83754469-0bce-45ca-adaf-20d0d29c34d8,1,deepseek-ai/deepseek-coder-1.3b-base," if ((arrayA[k] + arrayB[k]) %2 === 0 ){ -",248,0.952 +",248,0.952 2025-09-22T13:58:02.633112,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -16301,7 +16301,7 @@ class Problem { ArrayList ans = new ArrayList(Arrays.asList(odds_arr));"," return ans; } }",62.895600000047125,,Completions generated successfully. Ready for display in your IDE.,c040edc9-139b-4c7c-9d34-6e42e4c3fe66,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T13:58:02.957578,[1],"//Given the lengths of the three sides of a triangle. Return the area of // the triangle rounded to 2 decimal points if the three sides form a valid triangle. // Otherwise return -1 @@ -16320,7 +16320,7 @@ function triangle_area(a, b, c){ } ",323.93690000026254,,Completions generated successfully. Ready for display in your IDE.,a8cc557b-512d-4f33-9872-c42200110795,1,deepseek-ai/deepseek-coder-1.3b-base," console.log((s*(s-a)*(s-b)*(s-c))**(.5)); -",301,0.9794 +",301,0.9794 2025-09-22T13:58:03.020582,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -16360,7 +16360,7 @@ function int_to_mini_roman(number){ return output; } ",63.02609999693232,,Completions generated successfully. Ready for display in your IDE.,e19fe86c-56ea-4f97-9f54-2eb363dd08a9,1,deepseek-ai/deepseek-coder-1.3b-base," -",39,1.0 +",39,1.0 2025-09-22T13:58:03.185943,[1],"def correct_bracketing(brackets: str) -> bool: """""" brackets is a string of ""<"" and "">"". return True if every opening bracket has a corresponding closing bracket. @@ -16383,7 +16383,7 @@ function int_to_mini_roman(number){ if num_left_brackets < 0: return False"," ",165.1588000022457,,Completions generated successfully. Ready for display in your IDE.,dcc342eb-d63d-4450-b06f-e35fc726eb4a,1,deepseek-ai/deepseek-coder-1.3b-base," - return num_left_brackets == 0",142,1.0 + return num_left_brackets == 0",142,1.0 2025-09-22T13:58:03.344796,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -16419,7 +16419,7 @@ function int_to_mini_roman(number){ ",158.6825999984285,,Completions generated successfully. Ready for display in your IDE.,a3d37171-4ae1-4e33-b7a7-91cbadc2fe7a,1,deepseek-ai/deepseek-coder-1.3b-base," for i in file_name: -",131,1.0 +",131,1.0 2025-09-22T13:58:03.597649,[1],"//Given array of integers, return array in strange order. // Strange sorting, is when you start with the minimum value, // then maximum of the remaining integers, then minimum and so on. @@ -16446,7 +16446,7 @@ function strange_sort_list(lst){ return output; }",252.47980000131065,,Completions generated successfully. Ready for display in your IDE.,780f4be0-03e3-401a-85e7-d4c55ed68b80,1,deepseek-ai/deepseek-coder-1.3b-base," tmp.splice(tmp.indexOf(output[i]), 1); -",227,0.8942 +",227,0.8942 2025-09-22T13:58:04.012763,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -16477,7 +16477,7 @@ class Problem { } }",414.40090000105556,,Completions generated successfully. Ready for display in your IDE.,eb4e07e2-a6da-4755-820b-69088b4e1b57,1,deepseek-ai/deepseek-coder-1.3b-base," boolean allUpperCase = dict.keySet().stream().allMatch(k -> k.matches(""^[A-Z]+$"")); -",386,1.0 +",386,1.0 2025-09-22T13:58:04.491313,[1],"//You are given a non-empty array of positive integers. Return the greatest integer that is greater than // zero, and has a frequency greater than or equal to the value of the integer itself. // The frequency of an integer is the number of times it appears in the array. @@ -16514,7 +16514,7 @@ function search(lst){ //create hash table with key as numbers from list values and vals as their frequencies for (var i=0; i < lst.length; i ++) { -",454,0.9176 +",454,0.9176 2025-09-22T13:58:04.585330,[1],"from typing import List, Tuple def get_row(lst: List[List[int]], x: int) -> List[Tuple[int, int]]: @@ -16539,7 +16539,7 @@ def get_row(lst: List[List[int]], x: int) -> List[Tuple[int, int]]: res = [(i, j) for i in range(len(lst)) for j in range(len(lst[i])) if lst[i][j] == x] res.sort(key=lambda t: (t[0], -t[1]))"," ",94.50729999662144,,Completions generated successfully. Ready for display in your IDE.,cb4e2b6b-8dd5-4838-aef9-27a4e3cf67c7,1,deepseek-ai/deepseek-coder-1.3b-base," - return res",69,1.0 + return res",69,1.0 2025-09-22T13:58:04.688030,[1],"def make_palindrome(string: str) -> str: """""" Find the shortest palindrome that begins with a supplied string. Algorithm idea is simple: @@ -16563,7 +16563,7 @@ def get_row(lst: List[List[int]], x: int) -> List[Tuple[int, int]]: return string ",102.22950000024866,,Completions generated successfully. Ready for display in your IDE.,24d49b6e-636c-4a47-8b92-f101c7d84767,1,deepseek-ai/deepseek-coder-1.3b-base," if not string: -",79,1.0 +",79,1.0 2025-09-22T13:58:04.802864,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -16601,7 +16601,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",114.57959999825107,,Completions generated successfully. Ready for display in your IDE.,fcb60b0e-eac0-4eb5-9c37-5cb5d2a2a446,1,deepseek-ai/deepseek-coder-1.3b-base," even++; -",90,1.0 +",90,1.0 2025-09-22T13:58:04.893590,[1],"//Filter an input array of strings only for ones that contain given substring // >>> filter_by_substring([], ""a"") // [] @@ -16615,7 +16615,7 @@ function filter_by_substring(strings, substring){ }"," return filtered; }",90.29430000373395,,Completions generated successfully. Ready for display in your IDE.,ea368f2b-21ab-4524-8f6c-684932aa2598,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:58:04.988363,[1],"//Find the shortest palindrome that begins with a supplied string. // Algorithm idea is simple: // - Find the longest postfix of supplied string that is a palindrome. @@ -16640,7 +16640,7 @@ function make_palindrome(string){ } ",95.03000000404427,,Completions generated successfully. Ready for display in your IDE.,4d92a830-9b1c-44cd-bed4-56e6d2ba199a,1,deepseek-ai/deepseek-coder-1.3b-base," break; -",72,1.0 +",72,1.0 2025-09-22T13:58:05.199622,[1],"from typing import List, Optional def next_smallest(lst: List[int]) -> Optional[int]: @@ -16668,7 +16668,7 @@ def next_smallest(lst: List[int]) -> Optional[int]: return min2 if min1 != min2 else None ",210.81630000117002,,Completions generated successfully. Ready for display in your IDE.,a6341523-47c7-4009-8eb4-77fa47aa3d3e,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(2, len(lst)): -",185,1.0 +",185,1.0 2025-09-22T13:58:05.377038,[1],"from typing import List def max_element(l: List[int]) -> int: @@ -16683,7 +16683,7 @@ def max_element(l: List[int]) -> int: return max_elem ",177.56910000025528,,Completions generated successfully. Ready for display in your IDE.,3f14dfe2-1829-4e69-92c6-475408c184be,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(l)): -",155,1.0 +",155,1.0 2025-09-22T13:58:05.494930,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -16712,7 +16712,7 @@ function f(n){ return result; }",116.97819999972126,,Completions generated successfully. Ready for display in your IDE.,bad39326-5e9a-4c76-98ec-751497f0f51b,1,deepseek-ai/deepseek-coder-1.3b-base," sum += k; -",93,1.0 +",93,1.0 2025-09-22T13:58:05.693330,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -16741,7 +16741,7 @@ function next_smallest(lst){ }else{ return second_smallest; } -",175,0.9757 +",175,0.9757 2025-09-22T13:58:05.810384,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -16771,7 +16771,7 @@ function parse_nested_parens(paren_string){ ",117.05229999643052,,Completions generated successfully. Ready for display in your IDE.,e2d9b26f-0ed8-459b-8bba-3513d4d8d769,1,deepseek-ai/deepseek-coder-1.3b-base," level += 1; -",93,1.0 +",93,1.0 2025-09-22T13:58:05.924341,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -16813,7 +16813,7 @@ function compare_one(a, b){ } ",113.2556000011391,,Completions generated successfully. Ready for display in your IDE.,b9270a89-5012-4446-a773-1b18b83a5e90,1,deepseek-ai/deepseek-coder-1.3b-base," return undefined; -",83,0.9914 +",83,0.9914 2025-09-22T13:58:06.066514,[1],"def correct_bracketing(brackets: str) -> bool: """""" brackets is a string of ""<"" and "">"". return True if every opening bracket has a corresponding closing bracket. @@ -16837,7 +16837,7 @@ function compare_one(a, b){ ",142.09739999932935,,Completions generated successfully. Ready for display in your IDE.,a73ba4b2-9ff3-45d5-8506-4314503a2928,1,deepseek-ai/deepseek-coder-1.3b-base," if bracket == '<': -",120,1.0 +",120,1.0 2025-09-22T13:58:06.171919,[1],"//Given a non-empty array of integers lst. add the even elements that are at odd indices.. // Examples: // >>> add([4, 2, 6, 7]) @@ -16850,7 +16850,7 @@ function add(lst){ } }",},105.07179999694927,,Completions generated successfully. Ready for display in your IDE.,35adc72d-77c3-41cf-b2d4-550637416bfe,1,deepseek-ai/deepseek-coder-1.3b-base," return add; -",84,1.0 +",84,1.0 2025-09-22T13:58:06.333595,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -16879,7 +16879,7 @@ class Problem { });"," } }",161.34880000026897,,Completions generated successfully. Ready for display in your IDE.,b13b7098-2415-4bee-b33f-e313b47c354b,1,deepseek-ai/deepseek-coder-1.3b-base," return String.join("" "", split); -",136,0.9922 +",136,0.9922 2025-09-22T13:58:06.511736,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -16924,7 +16924,7 @@ class Problem { } }",178.0750000034459,,Completions generated successfully. Ready for display in your IDE.,05915032-b905-49c3-a42c-7b2101410b00,1,deepseek-ai/deepseek-coder-1.3b-base," ans.append(word + "" ""); -",152,0.9615 +",152,0.9615 2025-09-22T13:58:06.600428,[1],"//Write a function that accepts two arrays of strings and returns the array that has // total number of chars in the all strings of the array less than the other array. // if the two arrays have the same number of chars, return the first array. @@ -16955,7 +16955,7 @@ function total_match(lst1, lst2){ } ",88.44210000097519,,Completions generated successfully. Ready for display in your IDE.,27b8cdb6-b9a4-458f-bb04-a71718e13275,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:58:06.719961,[1],"//Write a function that takes a message, and encodes in such a // way that it swaps case of all letters, replaces all vowels in // the message with the letter that appears 2 places ahead of that @@ -16978,7 +16978,7 @@ function encode(message){ return result; }",119.50390000129119,,Completions generated successfully. Ready for display in your IDE.,ee9b0efd-d9db-4331-a244-8f200ba04f12,1,deepseek-ai/deepseek-coder-1.3b-base," var result= """"; -",93,0.951 +",93,0.951 2025-09-22T13:58:07.412064,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17018,7 +17018,7 @@ class Problem { private static final int[] primes = {7,9}; /** Returns the smallest number greater than n that is both prime and divisible by all of the numbers in the array. */ - public static long nextPrimeDivBy",667,0.8502 + public static long nextPrimeDivBy",667,0.8502 2025-09-22T13:58:07.558078,[1],"from typing import Tuple def even_odd_count(num: int) -> Tuple[int, int]: @@ -17040,7 +17040,7 @@ def even_odd_count(num: int) -> Tuple[int, int]: else: odd_count += 1"," ",146.13440000539413,,Completions generated successfully. Ready for display in your IDE.,cdce061f-dbca-47f7-b312-564ca492caba,1,deepseek-ai/deepseek-coder-1.3b-base," - return even_count, odd_count",124,0.9915 + return even_count, odd_count",124,0.9915 2025-09-22T13:58:07.674222,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -17079,7 +17079,7 @@ function valid_date(date){ } ",115.81909999949858,,Completions generated successfully. Ready for display in your IDE.,9682ff18-5e0b-477f-9343-0305a1e923ab,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",89,1.0 +",89,1.0 2025-09-22T13:58:07.825086,[1],"//The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fibfib(0) == 0 // fibfib(1) == 0 @@ -17105,7 +17105,7 @@ function fibfib(n){ ",150.68189999874448,,Completions generated successfully. Ready for display in your IDE.,a5a9b58d-f871-4186-9026-d167d0affcb7,1,deepseek-ai/deepseek-coder-1.3b-base," throw new Error(""Invalid input""); -",124,0.9798 +",124,0.9798 2025-09-22T13:58:07.904502,[1],"//Given a non-empty array of integers, return the sum of all of the odd elements that are in even positions. // Examples // >>> solution([5, 8, 7, 1]) @@ -17124,7 +17124,7 @@ function solution(lst){ } } return result;",,79.30389999819454,,Completions generated successfully. Ready for display in your IDE.,2ba674fb-15e5-4633-a3d4-60383c567fa1,1,deepseek-ai/deepseek-coder-1.3b-base," -}",53,1.0 +}",53,1.0 2025-09-22T13:58:08.058895,[1],"//You'll be given a string of words, and your task is to count the number // of boredoms. A boredom is a sentence that starts with the word ""I"". // Sentences are delimited by '.', '?' or '!'. @@ -17143,7 +17143,7 @@ function is_bored(S){ } ",153.64250000129687,,Completions generated successfully. Ready for display in your IDE.,80cca7ce-023c-4098-b390-57cedfabb518,1,deepseek-ai/deepseek-coder-1.3b-base," for (let line in lines) { -",134,0.9654 +",134,0.9654 2025-09-22T13:58:08.321977,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17179,14 +17179,14 @@ class Problem { } }",262.7929999944172,,Completions generated successfully. Ready for display in your IDE.,62e4a4f4-019c-4304-b5ac-081b62fc9606,1,deepseek-ai/deepseek-coder-1.3b-base," if (lst2.get(j) % 2 == 1) { -",235,0.9811 +",235,0.9811 2025-09-22T13:58:08.371968,[1],"//Return sorted unique elements in an array // >>> unique([5, 3, 5, 2, 3, 3, 9, 0, 123]) // [0, 2, 3, 5, 9, 123] function unique(l){ return l.filter(function(e, i, a){ return a.indexOf(e) === i; - }).sort(function(a, b){return a-b});",,49.85599999781698,,Completions generated successfully. Ready for display in your IDE.,f0936ff0-36f3-4b3e-8b50-9e9d89f7b4d1,1,deepseek-ai/deepseek-coder-1.3b-base,,21,0.1946 + }).sort(function(a, b){return a-b});",,49.85599999781698,,Completions generated successfully. Ready for display in your IDE.,f0936ff0-36f3-4b3e-8b50-9e9d89f7b4d1,1,deepseek-ai/deepseek-coder-1.3b-base,,21,0.1946 2025-09-22T13:58:08.568550,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -17258,7 +17258,7 @@ function minPath(grid, k){ return path; }",196.50609999371227,,Completions generated successfully. Ready for display in your IDE.,1270fd65-1483-40ad-b840-c8d0430fd2e9,1,deepseek-ai/deepseek-coder-1.3b-base," if(y < n - 1){ -",167,1.0 +",167,1.0 2025-09-22T13:58:08.743298,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17305,7 +17305,7 @@ class Problem { } }",174.8321000050055,,Completions generated successfully. Ready for display in your IDE.,46b76a53-63dc-4430-b985-3e5087335d32,1,deepseek-ai/deepseek-coder-1.3b-base," long units = level / capacity; -",146,0.9545 +",146,0.9545 2025-09-22T13:58:08.837489,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17336,7 +17336,7 @@ class Problem { boolean allUpperCase = dict.keySet().stream().allMatch(k -> k.matches(""^[A-Z]+$"")); return allLowerCase || allUpperCase;",},93.60680000099819,,Completions generated successfully. Ready for display in your IDE.,9e9cd8f7-4f77-4fd3-9908-38943a10ada4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:08.971252,[1],"//Return array of prime factors of given integer in the order from smallest to largest. // Each of the factors should be arrayed number of times corresponding to how many times it appeares in factorization. // Input number should be equal to the product of all factors @@ -17359,7 +17359,7 @@ function factorize(n){ } ",133.80659999529598,,Completions generated successfully. Ready for display in your IDE.,8cbf1f87-2ad8-45a4-b9ea-b9dc3ff2db9c,1,deepseek-ai/deepseek-coder-1.3b-base," n /= factor; -",108,1.0 +",108,1.0 2025-09-22T13:58:09.067953,[1],"from typing import List def exchange(lst1: List[int], lst2: List[int]) -> str: @@ -17387,7 +17387,7 @@ def exchange(lst1: List[int], lst2: List[int]) -> str: if count > 0: return ""NO"""," return ""YES""",96.20699999504723,,Completions generated successfully. Ready for display in your IDE.,01f3a789-81d1-403a-b76f-141dbe4598e7,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",72,1.0 +",72,1.0 2025-09-22T13:58:09.761170,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -17425,7 +17425,7 @@ function same_chars(s0, s1){ break; } - //loop through each character in string",669,0.9419 + //loop through each character in string",669,0.9419 2025-09-22T13:58:09.848279,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17455,7 +17455,7 @@ class Problem { } return res;",},86.0259999972186,,Completions generated successfully. Ready for display in your IDE.,ca0b785d-e1ee-4c5e-aabe-03c4e0ed3d8b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:58:10.145066,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17486,7 +17486,7 @@ class Problem { while (x % i == 0) { primes.add(i); } -",270,1.0 +",270,1.0 2025-09-22T13:58:10.240652,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -17535,7 +17535,7 @@ function check_dict_case(dict){ return true; }",95.42570000485284,,Completions generated successfully. Ready for display in your IDE.,d8c5fccc-e19c-48dd-8e79-43b9a3498362,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,0.9511 +",69,0.9511 2025-09-22T13:58:10.347324,[1],"//Given an array of positive integers x. return a sorted array of all // elements that hasn't any even digit. // Note: Returned array should be sorted in increasing order. @@ -17555,7 +17555,7 @@ function unique_digits(x){ } ",106.18790000444278,,Completions generated successfully. Ready for display in your IDE.,20eb3dd0-5480-4e7c-97de-0449f05b6124,1,deepseek-ai/deepseek-coder-1.3b-base," var result = []; -",80,0.9892 +",80,0.9892 2025-09-22T13:58:10.785568,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -17583,7 +17583,7 @@ function string_xor(a, b){ //convert the input to an integer value using parseint() method: for (var i = 0; i < a.length; i++) { -",415,0.953 +",415,0.953 2025-09-22T13:58:10.878210,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17615,7 +17615,7 @@ class Problem { } }",92.65210000012303,,Completions generated successfully. Ready for display in your IDE.,b2b55878-0bc8-4eb1-84a9-994b73ccf5b2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:10.998899,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17654,7 +17654,7 @@ class Problem { }"," } }",120.72969999280758,,Completions generated successfully. Ready for display in your IDE.,eb9dee46-5cbf-4f58-80e6-e2f49972304c,1,deepseek-ai/deepseek-coder-1.3b-base," return minSum; -",100,1.0 +",100,1.0 2025-09-22T13:58:11.160011,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17686,7 +17686,7 @@ class Problem { } }",160.93230000115,,Completions generated successfully. Ready for display in your IDE.,45014262-fcee-499c-96d9-11c2e2aad646,1,deepseek-ai/deepseek-coder-1.3b-base," result.add(0L); -",138,1.0 +",138,1.0 2025-09-22T13:58:11.316727,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -17741,7 +17741,7 @@ function by_length(arr){ return arr3; }",156.2476000035531,,Completions generated successfully. Ready for display in your IDE.,15af05e2-384d-4014-b4be-7abb4401a77e,1,deepseek-ai/deepseek-coder-1.3b-base," arr3.push(""Four""); -",130,1.0 +",130,1.0 2025-09-22T13:58:11.472203,[1],"from typing import List def move_one_ball(arr: List[int]) -> bool: @@ -17783,7 +17783,7 @@ def move_one_ball(arr: List[int]) -> bool: ",155.63880000263453,,Completions generated successfully. Ready for display in your IDE.,aab48e06-36d5-49b6-8bdf-0afa7d29a233,1,deepseek-ai/deepseek-coder-1.3b-base," return False else: - ",131,0.9597 + ",131,0.9597 2025-09-22T13:58:11.948072,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17815,7 +17815,7 @@ class Problem { } }",475.33739999926183,,Completions generated successfully. Ready for display in your IDE.,27ac147c-fbee-4a95-a3ce-5dca144f5e37,1,deepseek-ai/deepseek-coder-1.3b-base," if (!text.substring(left, left + 1).equalsIgnoreCase(text.substring(right, right + 1))) { -",349,0.9731 +",349,0.9731 2025-09-22T13:58:12.144784,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17883,7 +17883,7 @@ class Problem { } }",196.75640000059502,,Completions generated successfully. Ready for display in your IDE.,f0ab80a1-1284-4884-9991-8617c94475c9,1,deepseek-ai/deepseek-coder-1.3b-base," for (Long number : lst) { -",167,0.9708 +",167,0.9708 2025-09-22T13:58:12.391777,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17912,7 +17912,7 @@ class Problem { } }",246.07010000181617,,Completions generated successfully. Ready for display in your IDE.,ac18328a-5878-4c59-887e-82c7b79a7f41,1,deepseek-ai/deepseek-coder-1.3b-base," Arrays.sort(split, new Comparator() { -",222,1.0 +",222,1.0 2025-09-22T13:58:12.433788,[1],"//Given a string, find out how many distinct characters (regardless of case) does it consist of // >>> count_distinct_characters(""xyzXYZ"") // 3 @@ -17927,7 +17927,7 @@ function count_distinct_characters(string){ } return result.length; } -",41.9257999965339,,Completions generated successfully. Ready for display in your IDE.,8e36385e-8537-486c-b441-5c5ce7b7707d,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",41.9257999965339,,Completions generated successfully. Ready for display in your IDE.,8e36385e-8537-486c-b441-5c5ce7b7707d,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:58:12.583158,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -17989,7 +17989,7 @@ class Problem { } }",149.50690000114264,,Completions generated successfully. Ready for display in your IDE.,81222eff-5258-4431-b7af-08fc4a4c96bc,1,deepseek-ai/deepseek-coder-1.3b-base," String output = """"; -",122,0.9324 +",122,0.9324 2025-09-22T13:58:12.705009,[1],"from typing import List def incr_list(l: List[int]) -> List[int]: @@ -18003,7 +18003,7 @@ def incr_list(l: List[int]) -> List[int]: return res ",121.35729999863543,,Completions generated successfully. Ready for display in your IDE.,4de453a6-ecbd-4aab-aa3a-6499c912a98a,1,deepseek-ai/deepseek-coder-1.3b-base," for i in l: -",100,1.0 +",100,1.0 2025-09-22T13:58:12.975311,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18045,7 +18045,7 @@ class Problem { } }",270.20319999428466,,Completions generated successfully. Ready for display in your IDE.,75a12142-b74f-4c99-a99f-fdbf966a94b7,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < s4.length(); i++) { -",244,1.0 +",244,1.0 2025-09-22T13:58:13.153864,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18089,7 +18089,7 @@ class Problem { } }",177.95199999818578,,Completions generated successfully. Ready for display in your IDE.,e1f644f6-d2cb-4346-aebd-59f0e032f882,1,deepseek-ai/deepseek-coder-1.3b-base," if (currSum == 0) { -",152,1.0 +",152,1.0 2025-09-22T13:58:13.537631,[1],"from typing import List, Tuple def get_row(lst: List[List[int]], x: int) -> List[Tuple[int, int]]: @@ -18115,7 +18115,7 @@ def get_row(lst: List[List[int]], x: int) -> List[Tuple[int, int]]: ",383.5366999992402,,Completions generated successfully. Ready for display in your IDE.,b284f55d-4dfe-4d56-9280-bd5794b753ec,1,deepseek-ai/deepseek-coder-1.3b-base," res.sort(key=lambda x: (-x[0], len(lst)-x[0]-1-x[1])) -",359,0.9674 +",359,0.9674 2025-09-22T13:58:13.737420,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18146,7 +18146,7 @@ class Problem { boolean allUpperCase = dict.keySet().stream().allMatch(k -> k.matches(""^[A-Z]+$""));"," } }",199.4535999983782,,Completions generated successfully. Ready for display in your IDE.,98825a6e-3339-4f5a-b826-12227bc3fd49,1,deepseek-ai/deepseek-coder-1.3b-base," return allLowerCase || allUpperCase; -",168,1.0 +",168,1.0 2025-09-22T13:58:13.864132,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18179,7 +18179,7 @@ class Problem { }"," } }",126.23890000395477,,Completions generated successfully. Ready for display in your IDE.,e13397e9-d629-407e-8668-850c44693230,1,deepseek-ai/deepseek-coder-1.3b-base," return balance == 0; -",103,1.0 +",103,1.0 2025-09-22T13:58:13.989013,[1],"from typing import List def minPath(grid: List[List[int]], k: int) -> List[int]: @@ -18227,7 +18227,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: path.append(min_val) return path",125.18180000188295,,Completions generated successfully. Ready for display in your IDE.,f4c6f76f-ba57-43e5-9eac-33b1a8d5f72f,1,deepseek-ai/deepseek-coder-1.3b-base," row = i -",98,1.0 +",98,1.0 2025-09-22T13:58:14.095786,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -18259,7 +18259,7 @@ function correct_bracketing(brackets){ } }",106.51379999762867,,Completions generated successfully. Ready for display in your IDE.,838752e1-0414-4e8e-9ff3-943085bbeb4a,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",84,1.0 +",84,1.0 2025-09-22T13:58:14.139474,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -18283,7 +18283,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: groups.append(group) group = '' return groups -",43.07939999853261,,Completions generated successfully. Ready for display in your IDE.,4ce9a8b7-f31f-4e3c-b957-4e470298b9a4,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",43.07939999853261,,Completions generated successfully. Ready for display in your IDE.,4ce9a8b7-f31f-4e3c-b957-4e470298b9a4,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:58:14.405502,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18316,7 +18316,7 @@ class Problem { } }",265.9630999987712,,Completions generated successfully. Ready for display in your IDE.,25dacfa5-b158-46df-93d9-d3bb57fee529,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < lst.size(); ++i) { -",242,0.9903 +",242,0.9903 2025-09-22T13:58:14.498833,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18352,7 +18352,7 @@ class Problem { } return max;",},93.01899999991292,,Completions generated successfully. Ready for display in your IDE.,6bb63187-0b15-474c-b145-c8c8e0d92eb2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:14.583211,[1],"//From a given array of integers, generate an array of rolling maximum element found until given moment // in the sequence. // >>> rolling_max([1, 2, 3, 2, 3, 4, 2]) @@ -18368,7 +18368,7 @@ function rolling_max(numbers){ } ",84.81480000045849,,Completions generated successfully. Ready for display in your IDE.,0a1de400-6950-48f3-ae21-6c8d365f0ff4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:14.649732,[1],"//Return length of given string // >>> strlen("""") // 0 @@ -18376,7 +18376,7 @@ function rolling_max(numbers){ // 3 function strlen(string){ return string.length;",,65.53350000467617,,Completions generated successfully. Ready for display in your IDE.,bbb69aac-5c2e-4691-b888-423fa7b30553,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:58:14.791793,[1],"//Return the largest prime factor of n. Assume n > 1 and is not a prime. // >>> largest_prime_factor(13195) // 29 @@ -18393,7 +18393,7 @@ function largest_prime_factor(n){ return result; }",142.21670000551967,,Completions generated successfully. Ready for display in your IDE.,cc6f0156-6706-4028-882a-042ecce36e9c,1,deepseek-ai/deepseek-coder-1.3b-base," n /= result; -",120,1.0 +",120,1.0 2025-09-22T13:58:14.969116,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18423,7 +18423,7 @@ class Problem { return res; } }",177.1669000008842,,Completions generated successfully. Ready for display in your IDE.,4dc397a3-cb68-4d3c-8aa6-cbc496b9f39c,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList res = new ArrayList<>(); -",150,1.0 +",150,1.0 2025-09-22T13:58:15.013149,[1],"def fruit_distribution(s: str, n: int) -> int: """""" In this task, you will be given a string that represents a number of apples and oranges @@ -18449,7 +18449,7 @@ class Problem { return n - sum(fruits) -",43.967999998130836,,Completions generated successfully. Ready for display in your IDE.,ad0791cf-17f3-4db9-bb8b-9fe811ad30a6,1,deepseek-ai/deepseek-coder-1.3b-base,,20,0.8142 +",43.967999998130836,,Completions generated successfully. Ready for display in your IDE.,ad0791cf-17f3-4db9-bb8b-9fe811ad30a6,1,deepseek-ai/deepseek-coder-1.3b-base,,20,0.8142 2025-09-22T13:58:15.233095,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18493,7 +18493,7 @@ class Problem { } }",219.434699996782,,Completions generated successfully. Ready for display in your IDE.,6ff4ddc3-0855-4855-85e6-b49a17952e90,1,deepseek-ai/deepseek-coder-1.3b-base," char left = word.charAt(j - 1); -",193,1.0 +",193,1.0 2025-09-22T13:58:15.341690,[1],"//Your task is to write a function that returns true if a number x is a simple // power of n and false in other cases. // x is a simple power of n if n**int=x @@ -18522,7 +18522,7 @@ function is_simple_power(x, n){ } ",108.18730000028154,,Completions generated successfully. Ready for display in your IDE.,b7cf0bea-65a4-4aa3-9118-d9772a339297,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",83,1.0 +",83,1.0 2025-09-22T13:58:15.435419,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18562,7 +18562,7 @@ class Problem { } }",93.86110000195913,,Completions generated successfully. Ready for display in your IDE.,1ed4c6dc-a215-44fb-9dfe-4493a6744b85,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:15.519682,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18611,7 +18611,7 @@ class Problem { } return valid_date; }",,83.77370000380324,,Completions generated successfully. Ready for display in your IDE.,d32cb07b-6421-4890-ade2-ffa7a17d6370,1,deepseek-ai/deepseek-coder-1.3b-base," -}",56,1.0 +}",56,1.0 2025-09-22T13:58:15.841993,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -18666,7 +18666,7 @@ function by_length(arr){ return arr3; }",322.1233000003849,,Completions generated successfully. Ready for display in your IDE.,d28e8d40-cc16-4b17-b9ea-ccc011e868d3,1,deepseek-ai/deepseek-coder-1.3b-base," if((typeof (num) == 'number') && ((num >= 1)&&(num <= 9))){ -",296,0.9472 +",296,0.9472 2025-09-22T13:58:16.051891,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18700,7 +18700,7 @@ class Problem { } }",210.05199999490287,,Completions generated successfully. Ready for display in your IDE.,855e9cba-5e79-48f6-a62f-f673af53f646,1,deepseek-ai/deepseek-coder-1.3b-base," char c = brackets.charAt(i); -",173,1.0 +",173,1.0 2025-09-22T13:58:16.292085,[1],"from typing import List, Optional def prod_signs(arr: List[int]) -> Optional[int]: @@ -18736,7 +18736,7 @@ def prod_signs(arr: List[int]) -> Optional[int]: else: sign_arr.append(0)",,240.03130000346573,,Completions generated successfully. Ready for display in your IDE.,32952b3b-085b-42bf-bcac-941fc86d7d78,1,deepseek-ai/deepseek-coder-1.3b-base," - return abs(product(arr)) * sum(sign_arr)",213,0.9902 + return abs(product(arr)) * sum(sign_arr)",213,0.9902 2025-09-22T13:58:16.751593,[1],"from typing import Tuple def even_odd_palindrome(n: int) -> Tuple[int, int]: @@ -18766,7 +18766,7 @@ def even_odd_palindrome(n: int) -> Tuple[int, int]: sum(1 for i in range(1, n + 1) if str(i) == str(i)[::-1] and i % 2 == 0),"," ) ",459.36580000125105,,Completions generated successfully. Ready for display in your IDE.,20832017-9ff0-48ce-a78b-5d86963330d1,1,deepseek-ai/deepseek-coder-1.3b-base," sum(1 for i in range(1, n + 1) if str(i) == str(i)[::-1] and i % 2 != 0) -",430,0.9968 +",430,0.9968 2025-09-22T13:58:16.825346,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18795,7 +18795,7 @@ class Problem { }); return String.join("" "", split); }",,73.37289999850327,,Completions generated successfully. Ready for display in your IDE.,a8e9343a-e501-414d-969f-dbb6ee93b12d,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:58:16.923646,[1],"//Return true is array elements are monotonically increasing or decreasing. // >>> monotonic([1, 2, 4, 20]) // true @@ -18823,7 +18823,7 @@ function monotonic(l){ } ",97.68930000427645,,Completions generated successfully. Ready for display in your IDE.,2ebed83f-6898-45d5-ae43-351d8696bc51,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:58:17.071470,[1],"def largest_prime_factor(n: int) -> int: """"""Return the largest prime factor of n. Assume n > 1 and is not a prime. >>> largest_prime_factor(13195) @@ -18841,7 +18841,7 @@ function monotonic(l){ ",148.07099999597995,,Completions generated successfully. Ready for display in your IDE.,16ab4b38-7dd0-46ce-b4cd-03488c28e566,1,deepseek-ai/deepseek-coder-1.3b-base," if n % factor == 0: -",125,1.0 +",125,1.0 2025-09-22T13:58:17.159696,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18872,7 +18872,7 @@ class Problem { } }",87.51359999587294,,Completions generated successfully. Ready for display in your IDE.,6cfcc60c-e9a3-4ffa-8e02-fc8348f9baa9,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:17.343847,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -18919,7 +18919,7 @@ class Problem { } }",184.33509999886155,,Completions generated successfully. Ready for display in your IDE.,12819cad-98b4-47e3-9646-9e8a26e78d39,1,deepseek-ai/deepseek-coder-1.3b-base," for (long unit : well) { -",155,1.0 +",155,1.0 2025-09-22T13:58:17.479753,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -18969,7 +18969,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",135.127100002137,,Completions generated successfully. Ready for display in your IDE.,439af854-9c61-4325-98c5-65e19b02c64b,1,deepseek-ai/deepseek-coder-1.3b-base," return number > 1; -",108,1.0 +",108,1.0 2025-09-22T13:58:17.688733,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -19003,7 +19003,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",209.0978000051109,,Completions generated successfully. Ready for display in your IDE.,7a23b118-288d-47c0-9a8a-b2b8baf83aae,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(numbers)-1): -",182,1.0 +",182,1.0 2025-09-22T13:58:17.889684,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19047,7 +19047,7 @@ class Problem { } }",200.48120000137715,,Completions generated successfully. Ready for display in your IDE.,5f9967c0-1b8b-4763-aeba-8d7015252ed6,1,deepseek-ai/deepseek-coder-1.3b-base," return digits.toArray(new Long[]{}); -",172,0.9962 +",172,0.9962 2025-09-22T13:58:18.039747,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19078,7 +19078,7 @@ class Problem { } }",150.18549999513198,,Completions generated successfully. Ready for display in your IDE.,c03e7c70-1d2a-4bf3-83fb-79c907895816,1,deepseek-ai/deepseek-coder-1.3b-base," while (temp > 0) { -",124,1.0 +",124,1.0 2025-09-22T13:58:18.149056,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -19111,7 +19111,7 @@ class Problem { ",108.79659999773139,,Completions generated successfully. Ready for display in your IDE.,d4ed1cbe-794a-43f8-bb4a-a38e4415e8b4,1,deepseek-ai/deepseek-coder-1.3b-base," count = 0 -",84,1.0 +",84,1.0 2025-09-22T13:58:18.231086,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19145,7 +19145,7 @@ class Problem { } return open == 0;",},81.60049999423791,,Completions generated successfully. Ready for display in your IDE.,660d79c8-0f85-4100-8ceb-4fb5a8816b0e,1,deepseek-ai/deepseek-coder-1.3b-base," } -",58,1.0 +",58,1.0 2025-09-22T13:58:18.367031,[1],"//The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fib4(0) -> 0 // fib4(1) -> 0 @@ -19174,7 +19174,7 @@ function fib4(n){ ",136.11539999692468,,Completions generated successfully. Ready for display in your IDE.,8da24113-e2f8-45e6-8868-d335f920f44b,1,deepseek-ai/deepseek-coder-1.3b-base," return prev[3]; -",112,0.8948 +",112,0.8948 2025-09-22T13:58:18.464411,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19219,7 +19219,7 @@ class Problem { } }",97.39429999899585,,Completions generated successfully. Ready for display in your IDE.,765ca340-6433-4a26-a4d2-2b75e22aafe6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T13:58:18.662935,[1],"from typing import List def by_length(arr: List[int]) -> List[str]: @@ -19246,7 +19246,7 @@ def by_length(arr: List[int]) -> List[str]: return digits ",198.25990000390448,,Completions generated successfully. Ready for display in your IDE.,82ef2847-4304-4001-94fd-7d6e8c6ad81c,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(digits)): -",174,1.0 +",174,1.0 2025-09-22T13:58:18.841175,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -19288,7 +19288,7 @@ function compare_one(a, b){ } ",177.7224000034039,,Completions generated successfully. Ready for display in your IDE.,acb3b80d-49db-4c27-a502-3a33fb4a96d1,1,deepseek-ai/deepseek-coder-1.3b-base," if (a > b + """" ){ -",152,0.9242 +",152,0.9242 2025-09-22T13:58:18.940734,[1],"//sum_to_n is a function that sums numbers from 1 to n. // >>> sum_to_n(30) // 465 @@ -19307,7 +19307,7 @@ function sum_to_n(n){ }","} ",99.59890000027372,,Completions generated successfully. Ready for display in your IDE.,3f7cd686-5b89-4016-a016-0f04a6a32e3d,1,deepseek-ai/deepseek-coder-1.3b-base," return sum; -",73,1.0 +",73,1.0 2025-09-22T13:58:19.224961,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19334,7 +19334,7 @@ class Problem { } } }",283.90000000217697,,Completions generated successfully. Ready for display in your IDE.,ca2376f1-1c3a-4298-a751-303545e90300,1,deepseek-ai/deepseek-coder-1.3b-base," if (n < 0 || !NumberUtils.isCreatable(Long.toString(n))) { -",257,0.9078 +",257,0.9078 2025-09-22T13:58:19.348896,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19357,7 +19357,7 @@ class Problem { return answer; } }",123.52429999737069,,Completions generated successfully. Ready for display in your IDE.,3cbafb53-6dcb-410c-8f9a-3a79f27af326,1,deepseek-ai/deepseek-coder-1.3b-base," String answer = """"; -",83,1.0 +",83,1.0 2025-09-22T13:58:19.453653,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19380,7 +19380,7 @@ class Problem { } }",104.25640000175918,,Completions generated successfully. Ready for display in your IDE.,b03e3f93-11d2-4fde-9bbd-98d57bf14ede,1,deepseek-ai/deepseek-coder-1.3b-base," } else{ -",79,0.9171 +",79,0.9171 2025-09-22T13:58:19.863525,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19429,7 +19429,7 @@ class Problem { }",409.55290000420064,,Completions generated successfully. Ready for display in your IDE.,e39a891f-cbdc-4efb-9dd9-321777d9a0af,1,deepseek-ai/deepseek-coder-1.3b-base," Collections.sort(l_e); for (int i = 0; i < l_e.size(); i++) { -",379,1.0 +",379,1.0 2025-09-22T13:58:20.099332,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -19465,7 +19465,7 @@ class Problem { ",235.88320000271779,,Completions generated successfully. Ready for display in your IDE.,1a8c03b9-7c79-4242-a6a0-f62de9cfad1d,1,deepseek-ai/deepseek-coder-1.3b-base," e_name = file_name.split('.')[1] -",206,1.0 +",206,1.0 2025-09-22T13:58:20.194383,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19509,7 +19509,7 @@ class Problem { } }",94.68649999325862,,Completions generated successfully. Ready for display in your IDE.,64a6e405-8814-476b-b8a5-b8e80210c288,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:20.790109,[1],"//I think we all remember that feeling when the result of some long-awaited // event is finally known. The feelings and thoughts you have at that moment are // definitely worth noting down and comparing. @@ -19537,7 +19537,7 @@ function compare(game, guess){ const diff = Math.abs(guess[i]-game[i]); if (diff === 0 ){ -",570,0.9434 +",570,0.9434 2025-09-22T13:58:20.997926,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19568,7 +19568,7 @@ class Problem { int res_denom = x_denom * n_denom;"," } }",207.6874999984284,,Completions generated successfully. Ready for display in your IDE.,b44455a1-6e2a-4562-b23b-69814c061aa2,1,deepseek-ai/deepseek-coder-1.3b-base," return res_num % res_denom == 0; -",184,0.9852 +",184,0.9852 2025-09-22T13:58:21.159796,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19646,7 +19646,7 @@ class Problem { } }",161.12869999778923,,Completions generated successfully. Ready for display in your IDE.,09385545-063c-418a-9a90-e8a3fa43b94c,1,deepseek-ai/deepseek-coder-1.3b-base," if (num >= 4) { -",134,1.0 +",134,1.0 2025-09-22T13:58:21.317605,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -19669,7 +19669,7 @@ class Problem { } }",157.92859999783104,,Completions generated successfully. Ready for display in your IDE.,d7709e8c-c57a-4c23-8ef1-339ef22d0cca,1,deepseek-ai/deepseek-coder-1.3b-base," shift %= digits.length(); -",134,0.9364 +",134,0.9364 2025-09-22T13:58:21.465437,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -19736,7 +19736,7 @@ function numerical_letter_grade(grades){ } }",},147.3446000018157,,Completions generated successfully. Ready for display in your IDE.,8f7e9dc1-d93b-4c71-a709-06236a55d8ce,1,deepseek-ai/deepseek-coder-1.3b-base," return grades_array; -",119,1.0 +",119,1.0 2025-09-22T13:58:21.519239,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -19765,7 +19765,7 @@ function parse_nested_parens(paren_string){ } ",54.318599999533035,,Completions generated successfully. Ready for display in your IDE.,f16968bd-bc35-4458-841f-9b1a330799fa,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:58:21.637621,[1],"//Write a function that returns true if the object q will fly, and false otherwise. // The object q will fly if it's balanced (it is a palindromic array) and the sum of its elements is less than or equal the maximum possible weight w. // Example: @@ -19794,7 +19794,7 @@ function will_it_fly(q, w){ } return balanced && sum <= w; }",117.92630000127247,,Completions generated successfully. Ready for display in your IDE.,6fe0774a-405b-4b4c-9cb6-33d69c775277,1,deepseek-ai/deepseek-coder-1.3b-base," // Your code here... -",95,0.911 +",95,0.911 2025-09-22T13:58:21.889178,[1],"from typing import List def move_one_ball(arr: List[int]) -> bool: @@ -19835,7 +19835,7 @@ def move_one_ball(arr: List[int]) -> bool: return True ",251.3390999956755,,Completions generated successfully. Ready for display in your IDE.,6b39fc3c-f727-4a5f-bb1f-c8a1fc90fa59,1,deepseek-ai/deepseek-coder-1.3b-base," if (flag and arr[i-1] > arr[i]): -",225,0.9964 +",225,0.9964 2025-09-22T13:58:22.112650,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -19912,7 +19912,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",222.87879999930738,,Completions generated successfully. Ready for display in your IDE.,72337864-14f5-4373-a254-499edf3f0a8d,1,deepseek-ai/deepseek-coder-1.3b-base," b_tmp = b.replace(',', '.') -",192,1.0 +",192,1.0 2025-09-22T13:58:22.235247,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -19952,7 +19952,7 @@ function match_parens(lst){ } return 'No'; }",122.80230000033043,,Completions generated successfully. Ready for display in your IDE.,1d3942a2-fcf4-4f8d-b810-25c45daa1a0a,1,deepseek-ai/deepseek-coder-1.3b-base," const open = '('; -",99,1.0 +",99,1.0 2025-09-22T13:58:22.334358,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20030,7 +20030,7 @@ class Problem { } }",98.46310000284575,,Completions generated successfully. Ready for display in your IDE.,5965340b-0319-4a5a-829a-7dc0a90d1c47,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T13:58:22.439919,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -20068,7 +20068,7 @@ function is_prime(n){ } ",105.66059999837307,,Completions generated successfully. Ready for display in your IDE.,7a7f6ecd-672a-48e9-b6a2-146259e6a76f,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",82,1.0 +",82,1.0 2025-09-22T13:58:22.543496,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20104,7 +20104,7 @@ class Problem { } }",102.64590000588214,,Completions generated successfully. Ready for display in your IDE.,126ee1ce-7e94-40f3-925c-9e81cd564eca,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:22.699626,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20143,7 +20143,7 @@ class Problem { } }",155.7228999954532,,Completions generated successfully. Ready for display in your IDE.,f18552fe-f709-409b-9922-91f13c359ccb,1,deepseek-ai/deepseek-coder-1.3b-base," minSum = currSum; -",131,1.0 +",131,1.0 2025-09-22T13:58:22.903572,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20176,7 +20176,7 @@ class Problem { } }",204.19479999691248,,Completions generated successfully. Ready for display in your IDE.,542ddbbf-e064-4710-95b9-60a23a0bfb78,1,deepseek-ai/deepseek-coder-1.3b-base," char[] chars = word.toCharArray(); -",165,1.0 +",165,1.0 2025-09-22T13:58:22.998930,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20210,7 +20210,7 @@ class Problem { } }",94.83680000266759,,Completions generated successfully. Ready for display in your IDE.,21bd12b0-c38c-473f-8aed-23ed82285a07,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:23.166971,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20239,7 +20239,7 @@ class Problem { }"," } }",168.12940000090748,,Completions generated successfully. Ready for display in your IDE.,1d052d59-329c-4ab8-b686-44da17877bea,1,deepseek-ai/deepseek-coder-1.3b-base," return Collections.max(primes); -",144,0.993 +",144,0.993 2025-09-22T13:58:23.519312,[1],"//Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. // >>> fizz_buzz(50) // 0 @@ -20263,7 +20263,7 @@ function fizz_buzz(n){ // your code here... for (var i=0 ; i bool: """"""Return true if a given number is prime, and false otherwise. >>> is_prime(6) @@ -20288,7 +20288,7 @@ function fizz_buzz(n){ ",238.88119999901392,,Completions generated successfully. Ready for display in your IDE.,9db91061-8ee8-40de-886f-7923295e58e2,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(2, (n // 2) + 1): -",217,0.9969 +",217,0.9969 2025-09-22T13:58:23.884939,[1],"def fib4(n: int) -> int: """"""The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: fib4(0) -> 0 @@ -20315,7 +20315,7 @@ function fizz_buzz(n){ ",126.42849999974715,,Completions generated successfully. Ready for display in your IDE.,f45e1928-f138-4063-8102-26db0d806731,1,deepseek-ai/deepseek-coder-1.3b-base," elif n == 3: -",104,1.0 +",104,1.0 2025-09-22T13:58:24.056430,[1],"//Complete the function that takes two integers and returns // the product of their unit digits. // Assume the input is always valid. @@ -20333,7 +20333,7 @@ function multiply(a, b){ } ",171.17619999771705,,Completions generated successfully. Ready for display in your IDE.,9a98e25c-6b06-4ce0-a6bb-4f36f7597b11,1,deepseek-ai/deepseek-coder-1.3b-base,"; let bStr = b.toString(); -",150,0.9659 +",150,0.9659 2025-09-22T13:58:24.168829,[1],"//brackets is a string of ""<"" and "">"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""<"") @@ -20359,7 +20359,7 @@ function correct_bracketing(brackets){ } ",112.18920000101207,,Completions generated successfully. Ready for display in your IDE.,9f11a16e-c716-42e2-be7f-1e5a7d76151a,1,deepseek-ai/deepseek-coder-1.3b-base," opens--; -",94,1.0 +",94,1.0 2025-09-22T13:58:24.367519,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -20388,7 +20388,7 @@ function f(n){ return result; }",198.35869999951683,,Completions generated successfully. Ready for display in your IDE.,31b9459e-78f7-4cd6-bcb8-ccfaff96f677,1,deepseek-ai/deepseek-coder-1.3b-base," if((i % 2) === 0 ) { -",175,0.9632 +",175,0.9632 2025-09-22T13:58:24.518082,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -20456,7 +20456,7 @@ function do_algebra(operator, operand){ } ",150.50300000439165,,Completions generated successfully. Ready for display in your IDE.,2ec4a7f5-1cf5-42cd-947a-c0f0a9c1c0f3,1,deepseek-ai/deepseek-coder-1.3b-base," do_again = true; -",123,1.0 +",123,1.0 2025-09-22T13:58:24.633565,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -20488,7 +20488,7 @@ function correct_bracketing(brackets){ } }",114.76190000394126,,Completions generated successfully. Ready for display in your IDE.,c6eef92e-6a39-4ddc-9332-5416cc0c430a,1,deepseek-ai/deepseek-coder-1.3b-base," opens.pop(); -",92,1.0 +",92,1.0 2025-09-22T13:58:24.686506,[1],"def special_factorial(n: int) -> int: """"""The Brazilian factorial is defined as: brazilian_factorial(n) = n! * (n-1)! * (n-2)! * ... * 1! @@ -20509,7 +20509,7 @@ function correct_bracketing(brackets){ else: return n * special_factorial(n-1) * factorial(n-1) -",52.60239999915939,,Completions generated successfully. Ready for display in your IDE.,c503a4e3-1c00-4493-97ad-2b5b0f1ec356,1,deepseek-ai/deepseek-coder-1.3b-base,,31,1.0 +",52.60239999915939,,Completions generated successfully. Ready for display in your IDE.,c503a4e3-1c00-4493-97ad-2b5b0f1ec356,1,deepseek-ai/deepseek-coder-1.3b-base,,31,1.0 2025-09-22T13:58:24.864895,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20558,7 +20558,7 @@ class Problem { } }",178.84629999753088,,Completions generated successfully. Ready for display in your IDE.,e26d296e-13bf-4ab4-9b98-743100cda4e5,1,deepseek-ai/deepseek-coder-1.3b-base," String[] ss = test.split("" ""); -",152,1.0 +",152,1.0 2025-09-22T13:58:25.133062,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20594,7 +20594,7 @@ class Problem { } }",267.8186999983154,,Completions generated successfully. Ready for display in your IDE.,936efa52-f75d-43dd-9d5b-4ba64a4930b9,1,deepseek-ai/deepseek-coder-1.3b-base," if (lst1.get(i) % 2 != 0) { -",246,1.0 +",246,1.0 2025-09-22T13:58:25.199434,[1],"//In this problem, you will implement a function that takes two arrays of numbers, // and determines whether it is possible to perform an exchange of elements // between them to make lst1 an array of only even numbers. @@ -20632,7 +20632,7 @@ function exchange(lst1, lst2){ } } ",66.26149999647168,,Completions generated successfully. Ready for display in your IDE.,a70278c8-a7ee-4b4e-9544-9cfc852c59c0,1,deepseek-ai/deepseek-coder-1.3b-base," - ",48,1.0 + ",48,1.0 2025-09-22T13:58:25.384729,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20672,7 +20672,7 @@ class Problem { return arrayList; } }",185.02869999792892,,Completions generated successfully. Ready for display in your IDE.,3028eb77-4998-4bdf-9773-c1470dcea6fc,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList arrayList = new ArrayList<>(); -",162,1.0 +",162,1.0 2025-09-22T13:58:25.464602,[1],"//A simple program which should return the value of x if n is // a prime number and should return the value of y otherwise. // Examples: @@ -20690,7 +20690,7 @@ function x_or_y(n, x, y){ }"," return x; }",79.35390000056941,,Completions generated successfully. Ready for display in your IDE.,6ba55fcb-ee82-442f-a9db-a80ff79e37d6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",58,1.0 +",58,1.0 2025-09-22T13:58:25.558672,[1],"def is_happy(s: str) -> bool: """"""You are given a string s. Your task is to check if the string is happy or not. @@ -20718,7 +20718,7 @@ function x_or_y(n, x, y){ ",94.10689999640454,,Completions generated successfully. Ready for display in your IDE.,e47ecbea-acb5-4c35-a59a-24503f2a2be0,1,deepseek-ai/deepseek-coder-1.3b-base," return False -",72,1.0 +",72,1.0 2025-09-22T13:58:25.716930,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -20776,7 +20776,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",158.1506000002264,,Completions generated successfully. Ready for display in your IDE.,6f0b0538-8467-4e23-8ce0-e3081fbd24bf,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('D-') -",131,1.0 +",131,1.0 2025-09-22T13:58:25.883326,[1],"//pairs_sum_to_zero takes an array of integers as an input. // it returns true if there are two distinct elements in the array that // sum to zero, and false otherwise. @@ -20806,7 +20806,7 @@ function pairs_sum_to_zero(l){ } else { break } -",143,0.9374 +",143,0.9374 2025-09-22T13:58:26.026148,[1],"from typing import Tuple def even_odd_count(num: int) -> Tuple[int, int]: @@ -20829,7 +20829,7 @@ def even_odd_count(num: int) -> Tuple[int, int]: ",142.15079999848967,,Completions generated successfully. Ready for display in your IDE.,ff8b0756-3d95-4e60-864f-c582d8768dab,1,deepseek-ai/deepseek-coder-1.3b-base," digit = int(ch) -",119,1.0 +",119,1.0 2025-09-22T13:58:26.224127,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20868,7 +20868,7 @@ class Problem { } else { return false; } -",171,1.0 +",171,1.0 2025-09-22T13:58:26.439019,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20898,7 +20898,7 @@ class Problem { }"," } }",214.6400999990874,,Completions generated successfully. Ready for display in your IDE.,6f988f2c-6f31-4b19-97c5-686bbdf35eea,1,deepseek-ai/deepseek-coder-1.3b-base," return newString.toString(); -",114,0.9642 +",114,0.9642 2025-09-22T13:58:26.605572,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -20936,7 +20936,7 @@ function even_odd_palindrome(n){ } }",},166.84829999576323,,Completions generated successfully. Ready for display in your IDE.,9143ede2-8e0b-4bb8-9c09-1ce62d2c6a03,1,deepseek-ai/deepseek-coder-1.3b-base," console.log([even, odd]); -",141,1.0 +",141,1.0 2025-09-22T13:58:26.807051,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -20975,7 +20975,7 @@ function find_max(words){ return max_unique_word; }",200.90049999998882,,Completions generated successfully. Ready for display in your IDE.,193381ee-832a-48ce-a8e5-2258f99eb81f,1,deepseek-ai/deepseek-coder-1.3b-base," if (word <= max_unique_word) { -",172,0.9731 +",172,0.9731 2025-09-22T13:58:26.894595,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -20992,7 +20992,7 @@ class Problem { public static ArrayList getPositive(ArrayList l) { return (new ArrayList(l.stream().filter(x -> x > 0).collect(Collectors.toList())));",},87.36759999737842,,Completions generated successfully. Ready for display in your IDE.,cc4a10e5-d589-4463-9639-6460cfc43ea0,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:58:26.963870,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21024,7 +21024,7 @@ class Problem { } return isPalindrome; }",,68.94160000229022,,Completions generated successfully. Ready for display in your IDE.,2822b135-30bd-4fbc-a54d-c008ebb6e289,1,deepseek-ai/deepseek-coder-1.3b-base," -}",45,0.9511 +}",45,0.9511 2025-09-22T13:58:27.289353,[1],"//I think we all remember that feeling when the result of some long-awaited // event is finally known. The feelings and thoughts you have at that moment are // definitely worth noting down and comparing. @@ -21049,7 +21049,7 @@ function compare(game, guess){ ",325.1814999966882,,Completions generated successfully. Ready for display in your IDE.,90600093-62c2-4541-b798-97da3edb92a1,1,deepseek-ai/deepseek-coder-1.3b-base," const diff = Math.abs((guess[i]-game[i])) result.push(diff) -",300,0.9945 +",300,0.9945 2025-09-22T13:58:27.364604,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21096,7 +21096,7 @@ class Problem { return l_s; } }",75.6143000035081,,Completions generated successfully. Ready for display in your IDE.,13d0cfe3-af84-4821-aa71-2ff5a089c043,1,deepseek-ai/deepseek-coder-1.3b-base," -",46,0.9367 +",46,0.9367 2025-09-22T13:58:27.432545,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21124,7 +21124,7 @@ class Problem { } return boredoms; }",,67.17690000368748,,Completions generated successfully. Ready for display in your IDE.,2395cfb6-fb0b-4a10-91b0-b1acf02f44a2,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T13:58:27.681496,[1],"//Task // Write a function that takes a string as input and returns the sum of the upper characters only' // ASCII codes. @@ -21149,7 +21149,7 @@ function digitSum(s){ return sum; }",249.35069999628467,,Completions generated successfully. Ready for display in your IDE.,61d8f1b3-5d9f-41e9-8718-9439329f5081,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=0 ; i < s.length; ++i) { -",226,0.9532 +",226,0.9532 2025-09-22T13:58:28.106834,[1],"def modp(n: int, p: int) -> int: """"""Return 2^n modulo p (be aware of numerics). >>> modp(3, 5) @@ -21170,7 +21170,7 @@ function digitSum(s){ ",424.5112999997218,,Completions generated successfully. Ready for display in your IDE.,9420f713-d86a-43b9-a243-ee673ce863ce,1,deepseek-ai/deepseek-coder-1.3b-base," if not isinstance(n, int): raise TypeError(""modp: n must be an integer"") if not isinstance(p, int): -",402,0.9741 +",402,0.9741 2025-09-22T13:58:28.254430,[1],"//Create a function that takes a value (string) representing a number // and returns the closest integer to it. If the number is equidistant // from two integers, round it away from zero. @@ -21200,7 +21200,7 @@ function closest_integer(value){ } ",148.05069999420084,,Completions generated successfully. Ready for display in your IDE.,f5626f26-91d5-4cc9-ad85-ae1f0e358323,1,deepseek-ai/deepseek-coder-1.3b-base," return Math.floor(value); -",123,1.0 +",123,1.0 2025-09-22T13:58:28.361697,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -21226,7 +21226,7 @@ function is_multiply_prime(a){ } }",106.46429999906104,,Completions generated successfully. Ready for display in your IDE.,ed9a4dd0-17a5-47a4-87dd-d5e19700119f,1,deepseek-ai/deepseek-coder-1.3b-base," i++; -",83,1.0 +",83,1.0 2025-09-22T13:58:28.488965,[1],"//Write a function that accepts an array of strings as a parameter, // deletes the strings that have odd lengths from it, // and returns the resulted array with a sorted order, @@ -21270,7 +21270,7 @@ function sorted_list_sum(lst){ var a = [""aa"", ""a"", ""aaa""]; var b = [""ab"", ""a"", ""aaa"", ""cd""];",127.49160000385018,,Completions generated successfully. Ready for display in your IDE.,775f966f-94db-4fa4-b9bc-4071ee650e79,1,deepseek-ai/deepseek-coder-1.3b-base," return 1; -",100,1.0 +",100,1.0 2025-09-22T13:58:28.587756,[1],"//You are given an array of integers. // You need to find the largest prime value and return the sum of its digits. // Examples: @@ -21301,7 +21301,7 @@ function skjkasdkd(lst){ sum += Number(digits[i]);"," return sum; }",98.61800000362564,,Completions generated successfully. Ready for display in your IDE.,5aa019d9-6e68-4817-a943-aa5adee4a5b4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",75,1.0 +",75,1.0 2025-09-22T13:58:28.744750,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21337,7 +21337,7 @@ class Problem { }",156.32049999840092,,Completions generated successfully. Ready for display in your IDE.,1f0f29cf-8aff-473a-b196-ea7cf7217165,1,deepseek-ai/deepseek-coder-1.3b-base," long i = 5; -",132,1.0 +",132,1.0 2025-09-22T13:58:28.831304,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -21368,7 +21368,7 @@ function same_chars(s0, s1){ ",86.84019999782322,,Completions generated successfully. Ready for display in your IDE.,167ba9fb-7017-4772-b384-03e902eb47af,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:28.948524,[1],"from typing import Dict def check_dict_case(dict: Dict[str, str]) -> bool: @@ -21396,7 +21396,7 @@ def check_dict_case(dict: Dict[str, str]) -> bool: return False return True ",116.42969999957131,,Completions generated successfully. Ready for display in your IDE.,25cb8cf0-f93c-40e8-8d1b-0d3187b9df6f,1,deepseek-ai/deepseek-coder-1.3b-base," if not dict: -",84,1.0 +",84,1.0 2025-09-22T13:58:29.060232,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -21468,7 +21468,7 @@ function minPath(grid, k){ return path; }",111.86769999767421,,Completions generated successfully. Ready for display in your IDE.,b8066397-188d-4a81-985e-ff6dd7e13535,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:58:29.323687,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -21518,7 +21518,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",263.15559999784455,,Completions generated successfully. Ready for display in your IDE.,ca8ba7b8-dce9-4e48-a489-181643c00925,1,deepseek-ai/deepseek-coder-1.3b-base," if(number % i == 0 || number % (i + 2) == 0){ -",239,0.9693 +",239,0.9693 2025-09-22T13:58:29.391577,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21549,7 +21549,7 @@ class Problem { } return result; }",,67.28319999820087,,Completions generated successfully. Ready for display in your IDE.,7205709c-0058-4534-8d7a-5fc8dadd83ef,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:58:29.692673,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -21589,7 +21589,7 @@ function is_prime(n){ ",300.86080000182847,,Completions generated successfully. Ready for display in your IDE.,60568158-4630-4aa6-b375-f352b8b1a865,1,deepseek-ai/deepseek-coder-1.3b-base," //if n < 2 then it's not prime if (n<2) { -",278,0.9594 +",278,0.9594 2025-09-22T13:58:29.753416,[1],"from typing import List def sort_even(l: List[int]) -> List[int]: @@ -21607,7 +21607,7 @@ def sort_even(l: List[int]) -> List[int]: temp.append(l[i])"," for i in range(0, len(l), 2): l[i] = temp[i // 2] return l",60.85200000234181,,Completions generated successfully. Ready for display in your IDE.,6b59ec6b-28b5-4374-9a73-a7e935c13ff2,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T13:58:29.922325,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21648,7 +21648,7 @@ class Problem { } }",168.23430000658846,,Completions generated successfully. Ready for display in your IDE.,1a42a36f-48dd-49e7-aaf0-c6ac8971ce51,1,deepseek-ai/deepseek-coder-1.3b-base," for (Long i : good) { -",143,1.0 +",143,1.0 2025-09-22T13:58:30.060573,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -21725,7 +21725,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",138.3261000009952,,Completions generated successfully. Ready for display in your IDE.,148aedf7-dd57-4819-a8f1-ef3140b4e813,1,deepseek-ai/deepseek-coder-1.3b-base," return None -",88,1.0 +",88,1.0 2025-09-22T13:58:30.156142,[1],"//There are eight planets in our solar system: the closerst to the Sun // is Mercury, the next one is Venus, then Earth, Mars, Jupiter, Saturn, // Uranus, Neptune. @@ -21769,7 +21769,7 @@ function bf(planet1, planet2){ return res; }",94.94750000158092,,Completions generated successfully. Ready for display in your IDE.,66b4615f-d1be-4070-b3f7-30e2febca28c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:30.223268,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21800,7 +21800,7 @@ class Problem { } return inc || dec; }",,67.14269999793032,,Completions generated successfully. Ready for display in your IDE.,08c20b9e-4f4a-4fc2-a0ea-ad19c7986681,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T13:58:30.331711,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -21833,7 +21833,7 @@ class Problem { ",108.17540000425652,,Completions generated successfully. Ready for display in your IDE.,1f01371e-87f6-4f99-bcb2-ea9266dd337e,1,deepseek-ai/deepseek-coder-1.3b-base," j = 1 -",84,1.0 +",84,1.0 2025-09-22T13:58:30.511377,[1],"from typing import List def triples_sum_to_zero(l: List[int]) -> bool: @@ -21861,7 +21861,7 @@ def triples_sum_to_zero(l: List[int]) -> bool: return False ",179.77790000441018,,Completions generated successfully. Ready for display in your IDE.,edbf4995-df29-45eb-95fe-7476eb5851bc,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(l)): -",155,1.0 +",155,1.0 2025-09-22T13:58:30.606811,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21900,7 +21900,7 @@ class Problem { } return res;",},95.27630000229692,,Completions generated successfully. Ready for display in your IDE.,842fd954-70e6-4de3-bd1b-fb45ee8607ab,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:30.696685,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -21930,7 +21930,7 @@ class Problem { } }",89.84639999835053,,Completions generated successfully. Ready for display in your IDE.,9246213f-ac5e-472f-9d13-7c7520a1f962,1,deepseek-ai/deepseek-coder-1.3b-base," } -",65,1.0 +",65,1.0 2025-09-22T13:58:30.836555,[1],"from typing import List def max_fill(grid: List[List[int]], capacity: int) -> int: @@ -21967,7 +21967,7 @@ def max_fill(grid: List[List[int]], capacity: int) -> int: return times ",139.23119999526534,,Completions generated successfully. Ready for display in your IDE.,174d2a44-9ba9-4c91-8f90-108006de02ed,1,deepseek-ai/deepseek-coder-1.3b-base," for row in grid: -",114,1.0 +",114,1.0 2025-09-22T13:58:31.025090,[1],"//""Given an array representing a branch of a tree that has non-negative integer nodes // your task is to pluck one of the nodes and return it. // The plucked node should be the node with the smallest even value. @@ -22007,7 +22007,7 @@ function pluck(arr){ }else{ continue; } -",161,0.9825 +",161,0.9825 2025-09-22T13:58:31.104330,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -22039,7 +22039,7 @@ function pluck(arr){ return count ",79.1828999936115,,Completions generated successfully. Ready for display in your IDE.,6c0b4034-ddd6-4464-b8f6-31febd9a137d,1,deepseek-ai/deepseek-coder-1.3b-base," - ",43,1.0 + ",43,1.0 2025-09-22T13:58:31.215915,[1],"//You are given a non-empty array of positive integers. Return the greatest integer that is greater than // zero, and has a frequency greater than or equal to the value of the integer itself. // The frequency of an integer is the number of times it appears in the array. @@ -22074,7 +22074,7 @@ function search(lst){ }","} ",111.10719999851426,,Completions generated successfully. Ready for display in your IDE.,f16d3fbe-7314-445e-8dc5-a7f390752c9a,1,deepseek-ai/deepseek-coder-1.3b-base," return max; -",87,0.8919 +",87,0.8919 2025-09-22T13:58:31.330243,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22112,7 +22112,7 @@ class Problem { } }",114.8107999979402,,Completions generated successfully. Ready for display in your IDE.,f95e4171-cbf3-41f8-94aa-28241c1cb435,1,deepseek-ai/deepseek-coder-1.3b-base," max--; -",89,1.0 +",89,1.0 2025-09-22T13:58:31.417725,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22135,7 +22135,7 @@ class Problem { .filter(i -> s.charAt(i) == 'A' || s.charAt(i) == 'E' || s.charAt(i) == 'I' || s.charAt(i) == 'O' || s.charAt(i) == 'U') .count();",},86.75750000111293,,Completions generated successfully. Ready for display in your IDE.,28e8da89-518c-461e-8a25-079c407b7d79,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:31.502456,[1],"//You are given a string representing a sentence, // the sentence contains some words separated by a space, // and you have to return a string that contains the words from the original sentence, @@ -22165,7 +22165,7 @@ function words_in_sentence(sentence){ }).join(' '); }",84.92660000047181,,Completions generated successfully. Ready for display in your IDE.,a60f7fca-d87f-451d-b2e8-cad3af97ba20,1,deepseek-ai/deepseek-coder-1.3b-base," }; -",61,0.9555 +",61,0.9555 2025-09-22T13:58:31.687923,[1],"from typing import List def maximum(arr: List[int], k: int) -> List[int]: @@ -22206,7 +22206,7 @@ def maximum(arr: List[int], k: int) -> List[int]: arr.remove(max_num) return [max_num] + get_max(arr, k - 1) ",,185.44029999611666,,Completions generated successfully. Ready for display in your IDE.,d8cb5692-4f16-4e7b-971e-bb6cdb2854a7,1,deepseek-ai/deepseek-coder-1.3b-base," - return get_max(arr, k)",161,0.9776 + return get_max(arr, k)",161,0.9776 2025-09-22T13:58:31.845833,[1],"//Given a string, find out how many distinct characters (regardless of case) does it consist of // >>> count_distinct_characters(""xyzXYZ"") // 3 @@ -22222,7 +22222,7 @@ function count_distinct_characters(string){ } ",157.214799997746,,Completions generated successfully. Ready for display in your IDE.,416c93c7-3efe-4eb3-ac1f-a55c263eef49,1,deepseek-ai/deepseek-coder-1.3b-base," result.push(arr[i]); -",134,1.0 +",134,1.0 2025-09-22T13:58:32.214304,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -22262,7 +22262,7 @@ function valid_date(date){ } ",368.67509999865433,,Completions generated successfully. Ready for display in your IDE.,97d3ef58-fa08-4ffa-933c-188007bad191,1,deepseek-ai/deepseek-coder-1.3b-base," //split by - then convert each element into an integer using parseint() let date_arr = date.split('-'); -",338,0.8831 +",338,0.8831 2025-09-22T13:58:32.393449,[1],"//Filter given array of any jsthon values only for integers // >>> filter_integers([""a"", 3.14, 5]) // [5] @@ -22277,7 +22277,7 @@ function filter_integers(values){ } ",178.93920000642538,,Completions generated successfully. Ready for display in your IDE.,e1716827-bbfe-46c5-a861-3081efa479c0,1,deepseek-ai/deepseek-coder-1.3b-base," _.each(values, function (value) { -",156,1.0 +",156,1.0 2025-09-22T13:58:32.480699,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -22309,7 +22309,7 @@ function correct_bracketing(brackets){ } }",86.58100000320701,,Completions generated successfully. Ready for display in your IDE.,34b9918d-6709-4641-9b5b-b0054ef8b525,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:32.702521,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22338,7 +22338,7 @@ class Problem { } }",221.86669999791775,,Completions generated successfully. Ready for display in your IDE.,7aa5a75b-dcad-414f-a5ff-7e7f0d87fa4f,1,deepseek-ai/deepseek-coder-1.3b-base," long oranges = Long.parseLong(splitted[1]); -",197,1.0 +",197,1.0 2025-09-22T13:58:32.921044,[1],"def special_factorial(n: int) -> int: """"""The Brazilian factorial is defined as: brazilian_factorial(n) = n! * (n-1)! * (n-2)! * ... * 1! @@ -22363,7 +22363,7 @@ class Problem { elif n == 2: return 3 else: -",201,0.9824 +",201,0.9824 2025-09-22T13:58:33.067302,[1],"def int_to_mini_roman(number: int) -> str: """""" Given a positive integer, obtain its roman numeral equivalent as a string, @@ -22399,7 +22399,7 @@ class Problem { ",146.05270000174642,,Completions generated successfully. Ready for display in your IDE.,14ef690a-b8d7-452d-a45a-cf43c1197228,1,deepseek-ai/deepseek-coder-1.3b-base," 900: 'cm', -",125,1.0 +",125,1.0 2025-09-22T13:58:33.356600,[1],"//Write a function that takes an integer a and returns true // if this ingeger is a cube of some integer number. // Note: you may assume the input is always valid. @@ -22422,7 +22422,7 @@ function iscube(a){ return false; } }",289.21429999900283,,Completions generated successfully. Ready for display in your IDE.,a0535d34-6281-42f4-a130-c47f4bd2c386,1,deepseek-ai/deepseek-coder-1.3b-base," if (Math.pow((int)(Math.sqrt(a)),3) == Math.abs(a)){ -",268,0.9799 +",268,0.9799 2025-09-22T13:58:33.516942,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22467,7 +22467,7 @@ class Problem { } }",159.54010000132257,,Completions generated successfully. Ready for display in your IDE.,dfb4bbf9-f107-4cd3-a4a1-ac21261fb886,1,deepseek-ai/deepseek-coder-1.3b-base," if (sum <= w) { -",133,1.0 +",133,1.0 2025-09-22T13:58:33.687838,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22493,7 +22493,7 @@ class Problem { return levels; } }",170.92119999870192,,Completions generated successfully. Ready for display in your IDE.,12ff86b2-5ca7-4976-a7cc-e1a0a101f45f,1,deepseek-ai/deepseek-coder-1.3b-base," List levels = new ArrayList<>(); -",146,1.0 +",146,1.0 2025-09-22T13:58:34.394030,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -22530,7 +22530,7 @@ print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",705.6670000019949, temp_index = i temp2_index = j elif abs(numbers[i] - numbers[j]) > temp: - temp = abs(numbers[i] - numbers[j",678,0.9896 + temp = abs(numbers[i] - numbers[j",678,0.9896 2025-09-22T13:58:34.492856,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22574,7 +22574,7 @@ class Problem { } }",98.76919999805978,,Completions generated successfully. Ready for display in your IDE.,421faf44-362a-457a-8ae1-158d4e35a010,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T13:58:34.802793,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22607,7 +22607,7 @@ class Problem { }",310.1033999992069,,Completions generated successfully. Ready for display in your IDE.,050babb3-d124-451f-a963-53d1c66eec6d,1,deepseek-ai/deepseek-coder-1.3b-base," long value = hashmap.get(key).longValue(); if (value == 1) { -",284,0.9971 +",284,0.9971 2025-09-22T13:58:35.065197,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -22684,7 +22684,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",261.9316999989678,,Completions generated successfully. Ready for display in your IDE.,779019bb-9d33-4ffd-a411-48a373a2b9ef,1,deepseek-ai/deepseek-coder-1.3b-base," elif isinstance(a, float) and isinstance(b, int): -",233,1.0 +",233,1.0 2025-09-22T13:58:35.160306,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22717,7 +22717,7 @@ class Problem { } return sum;",},94.53619999840157,,Completions generated successfully. Ready for display in your IDE.,ee832366-b832-4f79-acb0-ba0ca943ba06,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:35.223912,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -22755,7 +22755,7 @@ function find_max(words){ } return max_unique_word; }",64.08689999807393,,Completions generated successfully. Ready for display in your IDE.,a6c1a49f-36f3-4259-a63b-d44c10f65a97,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T13:58:35.474065,[1],"//Out of array of strings, return the longest one. Return the first one in case of multiple // strings of the same length. Return undefined in case the input array is empty. // >>> longest([]) @@ -22776,7 +22776,7 @@ function longest(strings){ } ",249.7019000002183,,Completions generated successfully. Ready for display in your IDE.,a03a8dd2-1bf8-4f2c-bb45-52a2303b022e,1,deepseek-ai/deepseek-coder-1.3b-base," if ((result === null || result.length <= strings[i].length)) { -",226,0.9655 +",226,0.9655 2025-09-22T13:58:35.542246,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -22813,7 +22813,7 @@ function is_prime(n){ return true; } ",68.03909999871394,,Completions generated successfully. Ready for display in your IDE.,b709c03e-2db4-4b4c-a247-e96a15ce365b,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:58:35.785884,[1],"//From a supplied array of numbers (of length at least two) select and return two that are the closest to each // other and return them in order (smaller number, larger number). // >>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) @@ -22835,7 +22835,7 @@ function find_closest_elements(numbers){ } ",242.73469999752706,,Completions generated successfully. Ready for display in your IDE.,8537bf36-3bd5-4d53-8ef8-d0d940160c03,1,deepseek-ai/deepseek-coder-1.3b-base," closest = Math.abs(numbers[i] - numbers[j]); -",217,0.9856 +",217,0.9856 2025-09-22T13:58:35.907251,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22863,7 +22863,7 @@ class Problem { return count; } }",121.4442999989842,,Completions generated successfully. Ready for display in your IDE.,90a52062-a906-4295-8b90-f78552905b48,1,deepseek-ai/deepseek-coder-1.3b-base," long count = 0; -",94,1.0 +",94,1.0 2025-09-22T13:58:36.102837,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22931,7 +22931,7 @@ class Problem { } }",195.51249999494758,,Completions generated successfully. Ready for display in your IDE.,c857ed9b-356e-47d1-b790-a44b33320836,1,deepseek-ai/deepseek-coder-1.3b-base," long previous_number = 0L; -",157,0.9905 +",157,0.9905 2025-09-22T13:58:36.184739,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -22998,7 +22998,7 @@ class Problem { } } }",81.86569999816129,,Completions generated successfully. Ready for display in your IDE.,91bb740a-707c-43ef-9000-7462596a055e,1,deepseek-ai/deepseek-coder-1.3b-base," -",52,1.0 +",52,1.0 2025-09-22T13:58:36.294022,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -23029,7 +23029,7 @@ function same_chars(s0, s1){ ",108.5562000007485,,Completions generated successfully. Ready for display in your IDE.,dfc6360f-215f-4f35-aa9a-0b4ada191a9f,1,deepseek-ai/deepseek-coder-1.3b-base," return res; -",84,1.0 +",84,1.0 2025-09-22T13:58:36.392482,[1],"def circular_shift(x: int, shift: int) -> str: """"""Circular shift the digits of the integer x, shift the digits right by shift and return the result as a string. @@ -23045,7 +23045,7 @@ function same_chars(s0, s1){ ",98.77099999721395,,Completions generated successfully. Ready for display in your IDE.,bcf332af-bf68-42de-8693-67c1dbdece38,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",73,1.0 +",73,1.0 2025-09-22T13:58:36.556610,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23123,7 +23123,7 @@ class Problem { } }",163.74070000165375,,Completions generated successfully. Ready for display in your IDE.,d569ed94-6ae5-4807-bd2c-9a0ce6c18aac,1,deepseek-ai/deepseek-coder-1.3b-base," num -= 100; -",136,1.0 +",136,1.0 2025-09-22T13:58:36.975073,[1],"//Implement a function that takes an non-negative integer and returns an array of the first n // integers that are prime numbers and less than n. // for example: @@ -23156,7 +23156,7 @@ function count_up_to(n){ // check all odd nums up to sqrt(n), then add them as primes until we reach n let output = new Array(); -",393,0.8927 +",393,0.8927 2025-09-22T13:58:37.081041,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23179,7 +23179,7 @@ class Problem { } }",106.44219999812776,,Completions generated successfully. Ready for display in your IDE.,6a359dc6-32d2-4d57-bd0b-b706b9a2c9f7,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",83,1.0 +",83,1.0 2025-09-22T13:58:37.176537,[1],"//Given array of integers, return array in strange order. // Strange sorting, is when you start with the minimum value, // then maximum of the remaining integers, then minimum and so on. @@ -23206,7 +23206,7 @@ function strange_sort_list(lst){ return output; }",95.44420000020182,,Completions generated successfully. Ready for display in your IDE.,1ddc9e9d-2423-4f82-a574-e6d3114556b4,1,deepseek-ai/deepseek-coder-1.3b-base," return []; -",72,1.0 +",72,1.0 2025-09-22T13:58:37.460143,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23251,7 +23251,7 @@ class Problem { } }",282.94170000299346,,Completions generated successfully. Ready for display in your IDE.,4e1490a9-551d-4754-ba74-0982a5ee6302,1,deepseek-ai/deepseek-coder-1.3b-base," List words = Arrays.asList(sentence.split("" "")); -",257,0.9752 +",257,0.9752 2025-09-22T13:58:37.555061,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -23288,7 +23288,7 @@ function max_fill(grid, capacity){ } ",94.69009999884292,,Completions generated successfully. Ready for display in your IDE.,fc5bc54d-ada1-434c-bf0c-c748dd6212eb,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,0.9223 +",68,0.9223 2025-09-22T13:58:37.671917,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -23326,7 +23326,7 @@ function is_prime(n){ ","} ",116.92219999531517,,Completions generated successfully. Ready for display in your IDE.,fc31c80f-caf0-4edd-a540-043de4639ffd,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",94,1.0 +",94,1.0 2025-09-22T13:58:37.787494,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23364,7 +23364,7 @@ class Problem { } }",114.67350000020815,,Completions generated successfully. Ready for display in your IDE.,d63e4b9b-4ef0-4748-989e-b7cce8f6c36f,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",89,1.0 +",89,1.0 2025-09-22T13:58:37.990263,[1],"def encode(message: str) -> str: """""" Write a function that takes a message, and encodes in such a @@ -23391,7 +23391,7 @@ class Problem { return ch return ''.join(map(lambda ch: encode_vowel(swap_case(ch)), message)) ",202.98420000472106,,Completions generated successfully. Ready for display in your IDE.,9d1b3084-43d0-4bd0-90a8-f47b7e2465e2,1,deepseek-ai/deepseek-coder-1.3b-base," def swap_case(ch: str) -> str: -",176,0.9947 +",176,0.9947 2025-09-22T13:58:38.077803,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -23423,7 +23423,7 @@ function correct_bracketing(brackets){ } }",86.8244000012055,,Completions generated successfully. Ready for display in your IDE.,bb79e5c1-85be-4e15-86ff-bdef7dd4de77,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:58:38.783590,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23471,7 +23471,7 @@ class Problem { for (int i = 2; i < 50; ++i) { int count = 0; for (int j = 2; j < i; ++j) { - if (i % j == 0",679,0.9787 + if (i % j == 0",679,0.9787 2025-09-22T13:58:38.870659,[1],"//From a supplied array of numbers (of length at least two) select and return two that are the closest to each // other and return them in order (smaller number, larger number). // >>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) @@ -23493,7 +23493,7 @@ function find_closest_elements(numbers){ } ",87.45059999637306,,Completions generated successfully. Ready for display in your IDE.,2ae2bb2c-c7e8-43fa-a458-f5eb168fea23,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:58:39.040838,[1],"//You are given 2 words. You need to return true if the second word or any of its rotations is a substring in the first word // >>> cycpattern_check(""abcd"", ""abd"") // false @@ -23515,7 +23515,7 @@ function cycpattern_check(a, b){ }","} ",169.2934000020614,,Completions generated successfully. Ready for display in your IDE.,48d7941d-ab91-48ed-ab87-fc794eb8bf6f,1,deepseek-ai/deepseek-coder-1.3b-base," console.log('result: ', result); -",146,0.9349 +",146,0.9349 2025-09-22T13:58:39.126144,[1],"//For a given array of integers, return an array consisting of a sum and a product of all the integers in an array. // Empty sum should be equal to 0 and empty product should be equal to 1. // >>> sum_product([]) @@ -23534,7 +23534,7 @@ function sum_product(numbers){ } ",84.8236999954679,,Completions generated successfully. Ready for display in your IDE.,90f3b29f-2bc5-4d5a-b9bc-b24740a145bb,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:58:39.266410,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -23563,7 +23563,7 @@ function f(n){ return result; }",140.6563999989885,,Completions generated successfully. Ready for display in your IDE.,88484d99-ff2b-4b4e-8608-f5dd1cf70347,1,deepseek-ai/deepseek-coder-1.3b-base," result.push(sum); -",117,1.0 +",117,1.0 2025-09-22T13:58:39.359864,[1],"//Given a positive integer n, return a sorted array that has the odd numbers in collatz sequence. // The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined // as follows: start with any positive integer n. Then each term is obtained from the @@ -23606,7 +23606,7 @@ function get_odd_collatz(n){ } ",93.51819999574218,,Completions generated successfully. Ready for display in your IDE.,803d5a50-04de-4341-a843-a5202a3a9232,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:58:40.061202,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23644,7 +23644,7 @@ class Problem { while (!found && j < 5) { if ((vowels[j] == arr[i]) || (vowels[j+1] == arr[i])){ found = true; - ",671,0.9603 + ",671,0.9603 2025-09-22T13:58:40.137106,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23693,7 +23693,7 @@ class Problem { } return s; }",,75.90390000405023,,Completions generated successfully. Ready for display in your IDE.,c8980740-7ff0-42a0-bb06-908a9f7159ec,1,deepseek-ai/deepseek-coder-1.3b-base," -}",50,1.0 +}",50,1.0 2025-09-22T13:58:40.549208,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23755,7 +23755,7 @@ class Problem { } }",411.84620000422,,Completions generated successfully. Ready for display in your IDE.,20d623fb-b549-4d16-aba6-c76de911367b,1,deepseek-ai/deepseek-coder-1.3b-base," if ((Math.sqrt(number)).equals(Math.floor(Math.sqrt(number))) && !isPrime(number)) { -",383,0.904 +",383,0.904 2025-09-22T13:58:40.720598,[1],"def is_happy(s: str) -> bool: """"""You are given a string s. Your task is to check if the string is happy or not. @@ -23783,7 +23783,7 @@ class Problem { ",170.89589999523014,,Completions generated successfully. Ready for display in your IDE.,356f8c6d-4cb7-446c-88d7-e9392346b7f2,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(n - 2): -",148,0.9495 +",148,0.9495 2025-09-22T13:58:40.814126,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -23822,7 +23822,7 @@ function find_max(words){ return max_unique_word; }",93.29659999639262,,Completions generated successfully. Ready for display in your IDE.,7b295ec8-0282-45cc-bd37-33ef96f92762,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:40.909176,[1],"//Return median of elements in the array l. // >>> median([3, 1, 2, 4, 5]) // 3 @@ -23837,7 +23837,7 @@ function median(l){ } ",95.41280000121333,,Completions generated successfully. Ready for display in your IDE.,39c4c0e0-0169-4c87-9a9d-b7a9e7b36c45,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",74,0.959 +",74,0.959 2025-09-22T13:58:41.276075,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23863,7 +23863,7 @@ class Problem { } }",366.31969999871217,,Completions generated successfully. Ready for display in your IDE.,35411230-734e-40aa-a5b1-b57ad06081cd,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = Math.max(x,y); i >= Math.min(x,y)-1; --i) { -",343,0.9936 +",343,0.9936 2025-09-22T13:58:41.335579,[1],"//Your task is to implement a function that will simplify the expression // x * n. The function returns true if x * n evaluates to a whole number and false // otherwise. Both x and n, are string representation of a fraction, and have the following format, @@ -23882,7 +23882,7 @@ function simplify(x, n){ let nDenominator = parseInt(n.split('/')[1]); let xTimesN = xNumerator * nNumerator / (xDenominator * nDenominator); return Number.isInteger(xTimesN);",,59.478800001670606,,Completions generated successfully. Ready for display in your IDE.,80bf699e-7903-4fe2-b620-1750674fa9a4,1,deepseek-ai/deepseek-coder-1.3b-base," -}",37,1.0 +}",37,1.0 2025-09-22T13:58:41.465105,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -23915,7 +23915,7 @@ class Problem { } }",128.91279999894323,,Completions generated successfully. Ready for display in your IDE.,ad67021d-0cf9-45fb-9bba-a36d6762e278,1,deepseek-ai/deepseek-coder-1.3b-base," sb.append("" ""); -",104,1.0 +",104,1.0 2025-09-22T13:58:41.610769,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -23965,7 +23965,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",145.69399999891175,,Completions generated successfully. Ready for display in your IDE.,b563bc84-e45d-4617-96ee-3b50b16245c6,1,deepseek-ai/deepseek-coder-1.3b-base," a = a + b; -",119,0.9934 +",119,0.9934 2025-09-22T13:58:41.694165,[1],"//For a given number n, find the largest number that divides n evenly, smaller than n // >>> largest_divisor(15) // 5 @@ -23979,7 +23979,7 @@ function largest_divisor(n){ }"," return ld; }",83.4125999972457,,Completions generated successfully. Ready for display in your IDE.,f6fcfa81-feb5-48d2-a6ca-e06e0e3ce1b4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:58:41.804159,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24018,7 +24018,7 @@ class Problem { } }",109.2516999997315,,Completions generated successfully. Ready for display in your IDE.,5523be2a-7ca3-4c67-968f-4fcfe313faed,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:41.892373,[1],"//Write a function that takes a message, and encodes in such a // way that it swaps case of all letters, replaces all vowels in // the message with the letter that appears 2 places ahead of that @@ -24041,7 +24041,7 @@ function encode(message){ return result; }",88.74210000067251,,Completions generated successfully. Ready for display in your IDE.,9f2f2586-eb00-49eb-836e-41475bc40ce5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:42.020173,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24081,7 +24081,7 @@ class Problem { } }",126.93250000302214,,Completions generated successfully. Ready for display in your IDE.,f54b5f22-dae5-436d-9e93-2fef967ee662,1,deepseek-ai/deepseek-coder-1.3b-base," return 1; -",100,1.0 +",100,1.0 2025-09-22T13:58:42.121787,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24117,7 +24117,7 @@ class Problem { } }",101.57299999991665,,Completions generated successfully. Ready for display in your IDE.,866cb142-4428-4aad-a2f5-3f53308cc835,1,deepseek-ai/deepseek-coder-1.3b-base," } -",73,1.0 +",73,1.0 2025-09-22T13:58:42.260269,[1],"from typing import List def maximum(arr: List[int], k: int) -> List[int]: @@ -24159,7 +24159,7 @@ def maximum(arr: List[int], k: int) -> List[int]: return sorted(get_max(arr, k)) ",138.39899999584304,,Completions generated successfully. Ready for display in your IDE.,1408a2db-d2fa-45aa-96dc-5cac929ac901,1,deepseek-ai/deepseek-coder-1.3b-base," elif k == 1: -",113,0.9859 +",113,0.9859 2025-09-22T13:58:42.459322,[1],"//Create a function which returns the largest index of an element which // is not greater than or equal to the element immediately preceding it. If // no such element exists then return -1. The given array will not contain @@ -24181,7 +24181,7 @@ function can_arrange(arr){ } ",198.53319999674568,,Completions generated successfully. Ready for display in your IDE.,b5be9715-a897-488d-be96-6c5e660f2a7a,1,deepseek-ai/deepseek-coder-1.3b-base," max = Math.max(max, arr[i]); -",175,1.0 +",175,1.0 2025-09-22T13:58:42.640046,[1],"from typing import Dict def histogram(test: str) -> Dict[str, int]: @@ -24213,7 +24213,7 @@ def histogram(test: str) -> Dict[str, int]: return result ",180.93550000048708,,Completions generated successfully. Ready for display in your IDE.,7deb464b-a205-4edf-a5f1-d9c6365bf8df,1,deepseek-ai/deepseek-coder-1.3b-base," if letters[letter] > most_count: -",156,1.0 +",156,1.0 2025-09-22T13:58:42.738150,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24252,7 +24252,7 @@ class Problem { } return minSum;",},97.80429999955231,,Completions generated successfully. Ready for display in your IDE.,15fd7d74-6cd4-4752-bc74-a97c6cf30e35,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:43.084995,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24294,7 +24294,7 @@ class Problem { } }",346.35910000361037,,Completions generated successfully. Ready for display in your IDE.,ff1a3c4c-6500-470a-905b-b04cb8b48327,1,deepseek-ai/deepseek-coder-1.3b-base," if (!lst.contains(s + ""0"") && !lst.contains(s + ""-1"")) { -",319,0.9574 +",319,0.9574 2025-09-22T13:58:43.242056,[1],"from typing import List, Optional def prod_signs(arr: List[int]) -> Optional[int]: @@ -24329,7 +24329,7 @@ def prod_signs(arr: List[int]) -> Optional[int]: sign_arr.append(-1) else:"," return sum(map(abs, arr)) * product(sign_arr)",157.50870000192663,,Completions generated successfully. Ready for display in your IDE.,fb2eb877-3715-4d69-8eb3-4126f7a453e1,1,deepseek-ai/deepseek-coder-1.3b-base," sign_arr.append(0) -",134,1.0 +",134,1.0 2025-09-22T13:58:43.470168,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -24397,7 +24397,7 @@ function do_algebra(operator, operand){ } ",227.25419999915175,,Completions generated successfully. Ready for display in your IDE.,c378f314-7530-4980-81d9-1c8a7a9378ae,1,deepseek-ai/deepseek-coder-1.3b-base," operand.splice(count + 1, 1); -",197,1.0 +",197,1.0 2025-09-22T13:58:43.524383,[1],"def cycpattern_check(a: str, b: str) -> bool: """"""You are given 2 words. You need to return True if the second word or any of its rotations is a substring in the first word >>> cycpattern_check('abcd', 'abd') @@ -24419,7 +24419,7 @@ function do_algebra(operator, operand){ return True"," return False ",53.74380000284873,,Completions generated successfully. Ready for display in your IDE.,cab36592-2469-4065-b00d-ba5fb74b5059,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:58:43.619084,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -24445,7 +24445,7 @@ function is_multiply_prime(a){ } }",95.19970000110334,,Completions generated successfully. Ready for display in your IDE.,a0a5f016-82ab-4d4f-ada6-2f4327575f82,1,deepseek-ai/deepseek-coder-1.3b-base," -",54,1.0 +",54,1.0 2025-09-22T13:58:44.317373,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24475,7 +24475,7 @@ class Problem { /* Given an array of integers, return the sum of all the numbers in the array except the largest number. If there are multiple elements, return the sum of those two numbers. - >>> getSum",672,0.9191 + >>> getSum",672,0.9191 2025-09-22T13:58:44.543446,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -24519,7 +24519,7 @@ function compare_one(a, b){ }else { console.log('error'); } -",197,0.8892 +",197,0.8892 2025-09-22T13:58:44.639349,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -24560,7 +24560,7 @@ function int_to_mini_roman(number){ } ",95.83279999787919,,Completions generated successfully. Ready for display in your IDE.,2b83fb92-5bbc-44a5-b042-41457ad1e445,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:58:44.765604,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24622,7 +24622,7 @@ class Problem { } }",126.40169999940554,,Completions generated successfully. Ready for display in your IDE.,9508b8cd-0a04-4f4e-bdd0-8162549869ee,1,deepseek-ai/deepseek-coder-1.3b-base," } -",91,1.0 +",91,1.0 2025-09-22T13:58:44.999694,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24665,7 +24665,7 @@ class Problem { return false; } }",234.00190000393195,,Completions generated successfully. Ready for display in your IDE.,54419078-ca7c-479a-824f-46f99a45e1b9,1,deepseek-ai/deepseek-coder-1.3b-base," if (n < 0 || n > 99999L) { -",207,0.878 +",207,0.878 2025-09-22T13:58:45.231847,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24688,7 +24688,7 @@ class Problem { } }",231.13699999521486,,Completions generated successfully. Ready for display in your IDE.,66c31d6f-e41a-4996-ba71-190579d978d4,1,deepseek-ai/deepseek-coder-1.3b-base," answer += Character.toLowerCase(string.charAt(i)); -",207,1.0 +",207,1.0 2025-09-22T13:58:45.428705,[1],"//pairs_sum_to_zero takes an array of integers as an input. // it returns true if there are two distinct elements in the array that // sum to zero, and false otherwise. @@ -24716,7 +24716,7 @@ function pairs_sum_to_zero(l){ } ",196.88010000390932,,Completions generated successfully. Ready for display in your IDE.,cb26c683-8db3-40f6-9ff4-1618626d2229,1,deepseek-ai/deepseek-coder-1.3b-base," while (i < l.length-1) { -",172,0.9443 +",172,0.9443 2025-09-22T13:58:45.542592,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24750,7 +24750,7 @@ class Problem { return open == 0; } }",114.0088999964064,,Completions generated successfully. Ready for display in your IDE.,7c8d43cf-b448-420e-9412-9ace3384c124,1,deepseek-ai/deepseek-coder-1.3b-base," int open = 0; -",91,1.0 +",91,1.0 2025-09-22T13:58:45.649716,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24794,7 +24794,7 @@ class Problem { return firstDigit % 2 != 0 && lastDigit % 2 != 0; }).count();",},106.42860000371002,,Completions generated successfully. Ready for display in your IDE.,547b791c-f00d-4ffd-820c-b380b2c8ec41,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:58:45.840909,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24845,7 +24845,7 @@ class Problem { }",190.91330000082962,,Completions generated successfully. Ready for display in your IDE.,0c2c4a44-55d2-4a59-99b7-0dcd0f599f0e,1,deepseek-ai/deepseek-coder-1.3b-base," case 'e': case 'y': -",163,0.9843 +",163,0.9843 2025-09-22T13:58:46.030447,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -24895,7 +24895,7 @@ function prime_fib(n){ fibNum++; }",},189.73259999620495,,Completions generated successfully. Ready for display in your IDE.,f6412012-666f-4a01-8f7d-43034a8f1257,1,deepseek-ai/deepseek-coder-1.3b-base," return fib(fibNum - 1); -",161,0.9583 +",161,0.9583 2025-09-22T13:58:46.249960,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -24919,7 +24919,7 @@ function vowels_count(s){ } ",219.4740000049933,,Completions generated successfully. Ready for display in your IDE.,3f4ce031-8ab9-4112-8642-5b2d7965cb84,1,deepseek-ai/deepseek-coder-1.3b-base," const vowels = ['a','e','i','o','u']; -",196,0.9542 +",196,0.9542 2025-09-22T13:58:46.320931,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -24966,7 +24966,7 @@ class Problem { return l_s; } }",70.652100002917,,Completions generated successfully. Ready for display in your IDE.,c5311bfe-c82c-4185-8635-72a5d99c5fb8,1,deepseek-ai/deepseek-coder-1.3b-base," -",46,1.0 +",46,1.0 2025-09-22T13:58:46.448351,[1],"//Given an array arr of integers, find the minimum number of elements that // need to be changed to make the array palindromic. A palindromic array is an array that // is read the same backwards and forwards. In one change, you can change one element to any other element. @@ -24985,7 +24985,7 @@ function smallest_change(arr){ } return changes; }",126.79790000402136,,Completions generated successfully. Ready for display in your IDE.,57e5aa18-610f-4a9b-8cc4-118c4790e464,1,deepseek-ai/deepseek-coder-1.3b-base," let changes = 0; -",104,0.957 +",104,0.957 2025-09-22T13:58:46.539400,[1],"from typing import List def all_prefixes(string: str) -> List[str]: @@ -24996,7 +24996,7 @@ def all_prefixes(string: str) -> List[str]: prefixes = [] for i in range(len(string)): prefixes.append(string[:i+1])",,91.48909999930765,,Completions generated successfully. Ready for display in your IDE.,01dd7dec-015e-4191-a37b-0e94de67ac4e,1,deepseek-ai/deepseek-coder-1.3b-base," - return prefixes",69,0.9881 + return prefixes",69,0.9881 2025-09-22T13:58:46.695644,[1],"//For a given array of integers, return an array consisting of a sum and a product of all the integers in an array. // Empty sum should be equal to 0 and empty product should be equal to 1. // >>> sum_product([]) @@ -25016,7 +25016,7 @@ function sum_product(numbers){ ",155.33520000462886,,Completions generated successfully. Ready for display in your IDE.,8ce967bc-3fae-41f2-92c5-6a9e4a66e100,1,deepseek-ai/deepseek-coder-1.3b-base," // Your code here... -",131,0.9213 +",131,0.9213 2025-09-22T13:58:46.783189,[1],"//Your task is to write a function that returns true if a number x is a simple // power of n and false in other cases. // x is a simple power of n if n**int=x @@ -25045,7 +25045,7 @@ function is_simple_power(x, n){ } ",87.30240000295453,,Completions generated successfully. Ready for display in your IDE.,01fbbc23-64b3-4eea-8946-c44383a1abb0,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:46.966158,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25087,7 +25087,7 @@ class Problem { } }",183.19599999813363,,Completions generated successfully. Ready for display in your IDE.,dfb94214-842a-4f96-949d-58fe7ea51c94,1,deepseek-ai/deepseek-coder-1.3b-base," if (c2 < 0) break; -",152,1.0 +",152,1.0 2025-09-22T13:58:47.123173,[1],"def modp(n: int, p: int) -> int: """"""Return 2^n modulo p (be aware of numerics). >>> modp(3, 5) @@ -25105,7 +25105,7 @@ class Problem { raise ValueError(""modp: invalid p"") if n < 0: raise ValueError(""modp: invalid n"")",,156.63019999919925,,Completions generated successfully. Ready for display in your IDE.,cf6d6abd-cba4-49b3-aab4-1e5a710e8dc9,1,deepseek-ai/deepseek-coder-1.3b-base," - return pow(2, n, p)",133,1.0 + return pow(2, n, p)",133,1.0 2025-09-22T13:58:47.211847,[1],"//This function takes an array l and returns an array l' such that // l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal // to the values of the corresponding indicies of l, but sorted. @@ -25135,7 +25135,7 @@ function sort_third(l){ return x; }",88.16859999933513,,Completions generated successfully. Ready for display in your IDE.,d766edf9-1957-4587-9fdf-06114ae40d23,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:58:47.310993,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -25203,7 +25203,7 @@ function do_algebra(operator, operand){ } ",99.29729999566916,,Completions generated successfully. Ready for display in your IDE.,449e281e-4f6c-4905-a6aa-716223445732,1,deepseek-ai/deepseek-coder-1.3b-base," } -",72,0.9832 +",72,0.9832 2025-09-22T13:58:47.406691,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -25241,7 +25241,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",95.32689999468857,,Completions generated successfully. Ready for display in your IDE.,7a604917-dc49-4292-89bc-c692518b595a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:58:47.543618,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25283,7 +25283,7 @@ class Problem { return sb.toString(); } }",136.13059999624966,,Completions generated successfully. Ready for display in your IDE.,c8317808-0ad6-4c7a-9d92-31725a3e4d78,1,deepseek-ai/deepseek-coder-1.3b-base," boolean letterNotFound = true; -",111,1.0 +",111,1.0 2025-09-22T13:58:47.683649,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25361,7 +25361,7 @@ class Problem { } }",140.06169999629492,,Completions generated successfully. Ready for display in your IDE.,b3f39fac-4823-4bcd-9340-da7c877e428c,1,deepseek-ai/deepseek-coder-1.3b-base," res += ""x""; -",113,1.0 +",113,1.0 2025-09-22T13:58:47.884165,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25405,7 +25405,7 @@ class Problem { } }",200.45180000306573,,Completions generated successfully. Ready for display in your IDE.,5c352d55-e64e-49cb-84f1-d2c33d0ab8e8,1,deepseek-ai/deepseek-coder-1.3b-base," public long getLastDigit(long n) { -",173,1.0 +",173,1.0 2025-09-22T13:58:48.577448,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25435,7 +25435,7 @@ class Problem { if (remainder == 0) { isPrime = false; - ",669,0.9489 + ",669,0.9489 2025-09-22T13:58:48.745736,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25456,7 +25456,7 @@ class Problem { } }",167.24989999784157,,Completions generated successfully. Ready for display in your IDE.,b20335b1-9bde-471e-ad7d-ebc319f54315,1,deepseek-ai/deepseek-coder-1.3b-base," .filter(l2::contains) -",142,0.9954 +",142,0.9954 2025-09-22T13:58:49.000333,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -25490,7 +25490,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",254.72780000563944,,Completions generated successfully. Ready for display in your IDE.,f332f814-aa34-4322-9110-d17d98ef26ca,1,deepseek-ai/deepseek-coder-1.3b-base," elif abs(numbers[i] - numbers[j]) == temp: -",228,1.0 +",228,1.0 2025-09-22T13:58:49.068020,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -25557,7 +25557,7 @@ function do_algebra(operator, operand){ return answer; } ",67.29020000057062,,Completions generated successfully. Ready for display in your IDE.,497131bd-b032-42b6-b922-1918ffb27800,1,deepseek-ai/deepseek-coder-1.3b-base," -",41,1.0 +",41,1.0 2025-09-22T13:58:49.204153,[1],"//Write a function that takes a string and returns an ordered version of it. // Ordered version of string, is a string where all words (separated by space) // are replaced by a new word where all the characters arranged in @@ -25579,7 +25579,7 @@ function anti_shuffle(s){ result.push(temp.join("""")); });",},136.4735000024666,,Completions generated successfully. Ready for display in your IDE.,0dea1e00-396b-4c77-b4b6-40e6197e4a2b,1,deepseek-ai/deepseek-coder-1.3b-base," return result.join("" "") -",113,0.9713 +",113,0.9713 2025-09-22T13:58:49.291581,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -25611,7 +25611,7 @@ function correct_bracketing(brackets){ } }",86.87970000028145,,Completions generated successfully. Ready for display in your IDE.,ed37007f-a535-4343-99ce-c4afe0ce8cf6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:49.387849,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -25661,7 +25661,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",95.86829999898328,,Completions generated successfully. Ready for display in your IDE.,a984b1b5-21da-48dd-b700-03e091f30417,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:58:49.543966,[1],"from typing import List def pluck(arr: List[int]) -> List[int]: @@ -25705,7 +25705,7 @@ def pluck(arr: List[int]) -> List[int]: ",155.9516000052099,,Completions generated successfully. Ready for display in your IDE.,904a82f9-8dc0-4d7b-b5e0-214d6de6e4b1,1,deepseek-ai/deepseek-coder-1.3b-base," return [] else: - ",130,1.0 + ",130,1.0 2025-09-22T13:58:49.663406,[1],"//Given a string of words, return an array of words split on whitespace, if no whitespaces exists in the text you // should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the // alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 @@ -25734,7 +25734,7 @@ function split_words(txt){ } }",119.28250000346452,,Completions generated successfully. Ready for display in your IDE.,d54b8ef9-2fc1-472d-a738-7357a86de4da,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1; -",91,1.0 +",91,1.0 2025-09-22T13:58:49.763920,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25776,7 +25776,7 @@ class Problem { }); return result;",},100.04000000481028,,Completions generated successfully. Ready for display in your IDE.,74ef9307-4a37-4ab7-b45b-4a1a3111939c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:58:50.509525,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -25848,7 +25848,7 @@ function do_algebra(operator, operand){ operand.splice(count + 1, 1); operator.splice(count, 1); count = 0; - do_",685,1.0 + do_",685,1.0 2025-09-22T13:58:50.571680,[1],"//Checks if given string is a palindrome // >>> is_palindrome("""") // true @@ -25860,7 +25860,7 @@ function do_algebra(operator, operand){ // false function is_palindrome(text){ return text == text.split("""").reverse().join("""");",,62.08510000578826,,Completions generated successfully. Ready for display in your IDE.,50c32fb2-6aa4-4ae5-aa55-26042f10ec70,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:58:50.725742,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25892,7 +25892,7 @@ class Problem { } }",153.75619999394985,,Completions generated successfully. Ready for display in your IDE.,9c9ac3b2-051d-4299-8d29-d9b12ebd5fbc,1,deepseek-ai/deepseek-coder-1.3b-base," while (first < last) { -",129,1.0 +",129,1.0 2025-09-22T13:58:51.420021,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -25917,7 +25917,7 @@ class Problem { private static final Scanner scan = new Scanner(System.in); public static void main(String[] args){ int t = scan.nextInt(); - for(int i=0;ismallest && (isNaN(parseInt(lst[i]))|| parseInt(lst[i])>=0))&& !isNaN(parseInt(lst[i])) ){ -",587,0.9695 +",587,0.9695 2025-09-22T13:58:52.935805,[1],"//Given a positive integer n, return the product of the odd digits. // Return 0 if all digits are even. // For example: @@ -26079,7 +26079,7 @@ function digits(n){ } return prod;"," ",77.08300000376767,,Completions generated successfully. Ready for display in your IDE.,a62e7a5d-fd27-49b8-8bf1-c0a45f59987f,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:58:53.096759,[1],"from typing import Dict def histogram(test: str) -> Dict[str, int]: @@ -26111,7 +26111,7 @@ def histogram(test: str) -> Dict[str, int]: return result ",160.73840000171913,,Completions generated successfully. Ready for display in your IDE.,1cd738df-569a-4c3a-8965-3342cdb753e3,1,deepseek-ai/deepseek-coder-1.3b-base," most_count = letters[letter] -",135,1.0 +",135,1.0 2025-09-22T13:58:53.191337,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26153,7 +26153,7 @@ class Problem { } }",94.40680000261636,,Completions generated successfully. Ready for display in your IDE.,9c2e763b-fa1c-4dc8-8830-b6d78dc3e534,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:58:53.449353,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26185,7 +26185,7 @@ class Problem { } }",257.80179999856045,,Completions generated successfully. Ready for display in your IDE.,f181d517-09b6-45d2-99df-e19fa642a9db,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < numbers.size(); i++) { -",232,1.0 +",232,1.0 2025-09-22T13:58:53.543925,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -26224,7 +26224,7 @@ function find_max(words){ return max_unique_word; }",94.70140000485117,,Completions generated successfully. Ready for display in your IDE.,cba2d4ee-5733-48e9-827d-76f54f0f3ac4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:58:53.615931,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -26250,7 +26250,7 @@ function string_xor(a, b){ } result = arrayXOR.join(''); return result;",,71.54750000336207,,Completions generated successfully. Ready for display in your IDE.,8372c1f6-ff52-457f-9ffe-c4bd8d2a7d36,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:58:53.845299,[1],"from typing import List def mean_absolute_deviation(numbers: List[float]) -> float: @@ -26266,7 +26266,7 @@ def mean_absolute_deviation(numbers: List[float]) -> float: absolute_deviations = [abs(x - mean) for x in numbers]"," print(mean_absolute_deviation([1.0, 2.0, 3.0, 4.0]))",228.80840000289027,,Completions generated successfully. Ready for display in your IDE.,e725a90a-7557-4c5d-8d6e-b524fbdfcb10,1,deepseek-ai/deepseek-coder-1.3b-base," return sum(absolute_deviations) / len(numbers) -",206,0.9911 +",206,0.9911 2025-09-22T13:58:54.106200,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26306,7 +26306,7 @@ class Problem { return words.get(0); } }",260.62209999508923,,Completions generated successfully. Ready for display in your IDE.,bd61114e-db83-4360-853b-bed635922c30,1,deepseek-ai/deepseek-coder-1.3b-base," Map wordsToUniqueCharacters = new HashMap<>(); -",232,1.0 +",232,1.0 2025-09-22T13:58:54.170030,[1],"//Filter an input array of strings only for ones that start with a given prefix. // >>> filter_by_prefix([], ""a"") // [] @@ -26316,7 +26316,7 @@ function filter_by_prefix(strings, prefix){ return strings.filter( function(str){ return str.startsWith(prefix); });",,63.945500005502254,,Completions generated successfully. Ready for display in your IDE.,78d42d16-c6bb-45ed-8532-10c74bc828ce,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:58:54.276163,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26364,7 +26364,7 @@ class Problem { return new ArrayList(); }",},105.4803000006359,,Completions generated successfully. Ready for display in your IDE.,d0c301e0-c1d8-4b02-a07e-aa79d1c2360e,1,deepseek-ai/deepseek-coder-1.3b-base," } -",77,1.0 +",77,1.0 2025-09-22T13:58:54.334734,[1],"from typing import List, Optional def next_smallest(lst: List[int]) -> Optional[int]: @@ -26391,7 +26391,7 @@ def next_smallest(lst: List[int]) -> Optional[int]: min2 = lst[i] return min2 if min1 != min2 else None ",58.20970000058878,,Completions generated successfully. Ready for display in your IDE.,c2c3fde2-9930-427d-a7cc-9cd3d9a67025,1,deepseek-ai/deepseek-coder-1.3b-base," -",35,1.0 +",35,1.0 2025-09-22T13:58:54.464036,[1],"def int_to_mini_roman(number: int) -> str: """""" Given a positive integer, obtain its roman numeral equivalent as a string, @@ -26427,7 +26427,7 @@ def next_smallest(lst: List[int]) -> Optional[int]: ",129.96530000236817,,Completions generated successfully. Ready for display in your IDE.,fe8ed2fa-5f54-4824-9af7-68a1a0963d1f,1,deepseek-ai/deepseek-coder-1.3b-base," 9: 'ix', -",104,1.0 +",104,1.0 2025-09-22T13:58:54.651841,[1],"from typing import List def find_max(words: List[str]) -> str: @@ -26454,7 +26454,7 @@ def find_max(words: List[str]) -> str: return max_word ",186.75810000422643,,Completions generated successfully. Ready for display in your IDE.,cfd869b3-e9d4-4c87-804e-a745cf0ff5fc,1,deepseek-ai/deepseek-coder-1.3b-base," unique_num = len(set(word)) -",163,1.0 +",163,1.0 2025-09-22T13:58:54.758505,[1],"def starts_one_ends(n: int) -> int: """""" Given a positive integer n, return the count of the numbers of n-digit @@ -26466,7 +26466,7 @@ def find_max(words: List[str]) -> str: return count ",106.97939999954542,,Completions generated successfully. Ready for display in your IDE.,5a8d10b6-3d8c-45b9-bc88-e243b61e286f,1,deepseek-ai/deepseek-coder-1.3b-base," count = 0 -",83,1.0 +",83,1.0 2025-09-22T13:58:54.907910,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26498,7 +26498,7 @@ class Problem { }"," } }",149.11730000312673,,Completions generated successfully. Ready for display in your IDE.,862c9fcb-c36e-40a1-ba8b-a1ae8a38470d,1,deepseek-ai/deepseek-coder-1.3b-base," return isPalindrome; -",125,1.0 +",125,1.0 2025-09-22T13:58:55.066717,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -26556,7 +26556,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",158.73580000334186,,Completions generated successfully. Ready for display in your IDE.,1aac02c5-cc91-4d32-a35d-3c381b2dc47c,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('C-') -",132,1.0 +",132,1.0 2025-09-22T13:58:55.124260,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26633,7 +26633,7 @@ class Problem { } return res; } -}",57.02439999731723,,Completions generated successfully. Ready for display in your IDE.,651d16af-90c5-485c-8626-adbd0db3be67,1,deepseek-ai/deepseek-coder-1.3b-base,,30,1.0 +}",57.02439999731723,,Completions generated successfully. Ready for display in your IDE.,651d16af-90c5-485c-8626-adbd0db3be67,1,deepseek-ai/deepseek-coder-1.3b-base,,30,1.0 2025-09-22T13:58:55.239434,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26664,7 +26664,7 @@ class Problem { } }",115.03480000101263,,Completions generated successfully. Ready for display in your IDE.,92c77b4a-6245-42fa-8ac4-9aab7f63b8b4,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",89,0.9286 +",89,0.9286 2025-09-22T13:58:55.541916,[1],"//Find how many times a given substring can be found in the original string. Count overlaping cases. // >>> how_many_times("""", ""a"") // 0 @@ -26681,7 +26681,7 @@ function how_many_times(string, substring){ }",302.6100999995833,,Completions generated successfully. Ready for display in your IDE.,223bd8a6-7110-4dbb-a691-bdc954f048dd,1,deepseek-ai/deepseek-coder-1.3b-base," i = string.indexOf(substring, i); i += substring.length; -",279,0.996 +",279,0.996 2025-09-22T13:58:56.234936,[1],"def sort_numbers(numbers: str) -> str: """""" Input is a space-delimited string of numberals from 'zero' to 'nine'. Valid choices are 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight' and 'nine'. @@ -26695,7 +26695,7 @@ function how_many_times(string, substring){ return res.strip() ",692.2431999992114,,Completions generated successfully. Ready for display in your IDE.,2c6643fe-6c14-4698-8468-cc3515ef59a2,1,deepseek-ai/deepseek-coder-1.3b-base," - mapper = {'zero': 0, 'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6, 'seven': 7, 'eight': 8, 'nine':",668,0.9952 + mapper = {'zero': 0, 'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6, 'seven': 7, 'eight': 8, 'nine':",668,0.9952 2025-09-22T13:58:56.348177,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26734,7 +26734,7 @@ class Problem { } }",112.69959999481216,,Completions generated successfully. Ready for display in your IDE.,dd442640-964b-4e30-b4b5-30588a5e3325,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",89,1.0 +",89,1.0 2025-09-22T13:58:56.813267,[1],"//Return a greatest common divisor of two integers a and b // >>> greatest_common_divisor(3, 5) // 1 @@ -26752,7 +26752,7 @@ function greatest_common_divisor(a, b){ for (i = 1; i <= min/2; ++i ){ if ((a%i==0) && (b%i==0)) return i } -",443,0.9674 +",443,0.9674 2025-09-22T13:58:56.900833,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26782,7 +26782,7 @@ class Problem { } }",87.90840000438038,,Completions generated successfully. Ready for display in your IDE.,c862ace2-31bf-41c9-97d9-a8e96885c15b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:58:57.126112,[1],"//Return a string containing space-delimited numbers starting from 0 upto n inclusive. // >>> string_sequence(0) // ""0"" @@ -26795,7 +26795,7 @@ function string_sequence(n){ } ",225.03190000134055,,Completions generated successfully. Ready for display in your IDE.,f21c9b96-eae9-4a33-9d50-a8412629bc2b,1,deepseek-ai/deepseek-coder-1.3b-base," for (let i = 0; i <= n ; i++ ){ -",203,0.9619 +",203,0.9619 2025-09-22T13:58:57.396817,[1],"def digits(n: int) -> int: """"""Given a positive integer n, return the product of the odd digits. Return 0 if all digits are even. @@ -26810,7 +26810,7 @@ function string_sequence(n){ if n < 10: return n if n % 2 == 1 else 0 d, m = divmod(n, 10)",,269.9012000011862,,Completions generated successfully. Ready for display in your IDE.,92876fe1-6e5c-4d19-8318-ba6014d8f3ef,1,deepseek-ai/deepseek-coder-1.3b-base," - return (m if m % 2 == 1 else 0) + digits(d)",247,0.9861 + return (m if m % 2 == 1 else 0) + digits(d)",247,0.9861 2025-09-22T13:58:57.546098,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26850,7 +26850,7 @@ class Problem { } }",149.54649999708636,,Completions generated successfully. Ready for display in your IDE.,5a89816e-0317-4beb-a634-17ff74a38d67,1,deepseek-ai/deepseek-coder-1.3b-base," arr.add(2l); -",126,0.9899 +",126,0.9899 2025-09-22T13:58:57.710100,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -26922,7 +26922,7 @@ function minPath(grid, k){ return path; }",163.79919999599224,,Completions generated successfully. Ready for display in your IDE.,d16b75bc-b4c2-436a-bd5b-a341bbad6626,1,deepseek-ai/deepseek-coder-1.3b-base," if(y > 0){ -",135,1.0 +",135,1.0 2025-09-22T13:58:57.826312,[1],"//The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fibfib(0) == 0 // fibfib(1) == 0 @@ -26948,7 +26948,7 @@ function fibfib(n){ ",115.57000000175321,,Completions generated successfully. Ready for display in your IDE.,12e6d977-3f61-4336-a7f8-69a444d0d07e,1,deepseek-ai/deepseek-coder-1.3b-base," return 1; -",93,1.0 +",93,1.0 2025-09-22T13:58:57.914824,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -26976,7 +26976,7 @@ class Problem { } }",88.58049999980722,,Completions generated successfully. Ready for display in your IDE.,f4af1f8e-b625-4844-8c9c-b854b10bfe55,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:58:58.040365,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27014,7 +27014,7 @@ class Problem { } }",125.88580000010552,,Completions generated successfully. Ready for display in your IDE.,53da4a36-a15e-4268-a6f9-3c9d90958aea,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",97,1.0 +",97,1.0 2025-09-22T13:58:58.166717,[1],"//For a given array of integers, return an array consisting of a sum and a product of all the integers in an array. // Empty sum should be equal to 0 and empty product should be equal to 1. // >>> sum_product([]) @@ -27033,7 +27033,7 @@ function sum_product(numbers){ } ",125.1923000017996,,Completions generated successfully. Ready for display in your IDE.,259cea27-9e79-454f-82dd-677292ff59cb,1,deepseek-ai/deepseek-coder-1.3b-base," let product = 1; -",104,1.0 +",104,1.0 2025-09-22T13:58:58.283548,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27061,7 +27061,7 @@ class Problem { } }",116.73180000070715,,Completions generated successfully. Ready for display in your IDE.,339cbd78-665b-4ba4-b92e-bb6812844238,1,deepseek-ai/deepseek-coder-1.3b-base," boredoms++; -",93,1.0 +",93,1.0 2025-09-22T13:58:58.369163,[1],"//Return the largest prime factor of n. Assume n > 1 and is not a prime. // >>> largest_prime_factor(13195) // 29 @@ -27078,7 +27078,7 @@ function largest_prime_factor(n){ return result; }",85.46989999740617,,Completions generated successfully. Ready for display in your IDE.,378a731f-9937-45e1-b6a8-6341333fb709,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:58:58.448202,[1],"//You are given a 2 dimensional data, as a nested arrays, // which is similar to matrix, however, unlike matrices, // each row may contain a different number of columns. @@ -27107,7 +27107,7 @@ function get_row(lst, x){ return coordinates.sort(function(a, b){ return a[0] - b[0] || b[1] - a[1]; });",,79.17239999369485,,Completions generated successfully. Ready for display in your IDE.,c843f4d8-7a49-4cf7-b140-fe064897051a,1,deepseek-ai/deepseek-coder-1.3b-base," -}",55,1.0 +}",55,1.0 2025-09-22T13:58:58.567373,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27169,7 +27169,7 @@ class Problem { } }",118.32039999717381,,Completions generated successfully. Ready for display in your IDE.,c3d42b62-59ea-4e3d-9dff-aee82f30c952,1,deepseek-ai/deepseek-coder-1.3b-base," } -",91,1.0 +",91,1.0 2025-09-22T13:58:58.695978,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27212,7 +27212,7 @@ class Problem { }"," } }",128.81689999630908,,Completions generated successfully. Ready for display in your IDE.,7d3d4afd-00ad-49cd-bd10-01015b5c8107,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",98,1.0 +",98,1.0 2025-09-22T13:58:58.918908,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27290,7 +27290,7 @@ class Problem { } }",222.77349999785656,,Completions generated successfully. Ready for display in your IDE.,2931f089-4935-4b22-9018-ec12616281db,1,deepseek-ai/deepseek-coder-1.3b-base," if (number < 1 || number > 3999) -",195,0.9922 +",195,0.9922 2025-09-22T13:58:58.983101,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -27323,7 +27323,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",63.37789999815868,,Completions generated successfully. Ready for display in your IDE.,007e375a-480f-4a1d-ab06-842a778acc5d,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:58:59.100843,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -27391,7 +27391,7 @@ function do_algebra(operator, operand){ } ",117.53260000114096,,Completions generated successfully. Ready for display in your IDE.,2674aca8-e546-4595-95ec-ff9ac76b5939,1,deepseek-ai/deepseek-coder-1.3b-base," break; -",83,1.0 +",83,1.0 2025-09-22T13:58:59.220187,[1],"from typing import Tuple def even_odd_count(num: int) -> Tuple[int, int]: @@ -27414,7 +27414,7 @@ def even_odd_count(num: int) -> Tuple[int, int]: ",119.26559999847086,,Completions generated successfully. Ready for display in your IDE.,dff9adea-e0b8-4261-a08f-7d413895df11,1,deepseek-ai/deepseek-coder-1.3b-base," even_count += 1 -",98,1.0 +",98,1.0 2025-09-22T13:58:59.461771,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27441,7 +27441,7 @@ class Problem { } }",241.68140000256244,,Completions generated successfully. Ready for display in your IDE.,e0af103f-f5b0-4ab5-9167-7e7deba93951,1,deepseek-ai/deepseek-coder-1.3b-base," return fib(n - 1) + fib(n - 2); -",217,1.0 +",217,1.0 2025-09-22T13:58:59.560162,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27490,7 +27490,7 @@ class Problem { } }",97.70220000064,,Completions generated successfully. Ready for display in your IDE.,1b60bd84-ec2d-43c2-9bbf-95b68dd552fd,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T13:59:00.088064,[1],"//""Given an array representing a branch of a tree that has non-negative integer nodes // your task is to pluck one of the nodes and return it. // The plucked node should be the node with the smallest even value. @@ -27530,7 +27530,7 @@ function pluck(arr){ result = [[arr[i], i]] } else { (index === -1 && result[0][1] > i)? null : result.push([arr[i], i]); -",503,0.9315 +",503,0.9315 2025-09-22T13:59:00.306699,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27563,7 +27563,7 @@ class Problem { } }",218.17490000103135,,Completions generated successfully. Ready for display in your IDE.,96335bac-351e-4be9-99d8-950b1b2232c2,1,deepseek-ai/deepseek-coder-1.3b-base," Arrays.sort(arr); int first = 0; -",193,0.9555 +",193,0.9555 2025-09-22T13:59:00.408656,[1],"from typing import List def move_one_ball(arr: List[int]) -> bool: @@ -27604,7 +27604,7 @@ def move_one_ball(arr: List[int]) -> bool: elif arr[i-1] < arr[i]:"," return True ",101.32290000183275,,Completions generated successfully. Ready for display in your IDE.,882a207c-5e73-44b8-9477-953def72e951,1,deepseek-ai/deepseek-coder-1.3b-base," continue -",69,0.9848 +",69,0.9848 2025-09-22T13:59:00.540516,[1],"from typing import List, Tuple, Optional def largest_smallest_integers(lst: List[int]) -> Tuple[Optional[int], Optional[int]]: @@ -27632,7 +27632,7 @@ def largest_smallest_integers(lst: List[int]) -> Tuple[Optional[int], Optional[i return (largest_negative, smallest_positive) ",131.66640000417829,,Completions generated successfully. Ready for display in your IDE.,f57d4881-2169-43aa-ac0f-ea648ad4572a,1,deepseek-ai/deepseek-coder-1.3b-base," for num in lst: -",109,1.0 +",109,1.0 2025-09-22T13:59:00.652850,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27668,7 +27668,7 @@ class Problem { }"," } }",112.50300000392599,,Completions generated successfully. Ready for display in your IDE.,b22ef6e8-ee05-4f14-b08f-248ad3eda1ab,1,deepseek-ai/deepseek-coder-1.3b-base," return count; -",88,1.0 +",88,1.0 2025-09-22T13:59:00.767427,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27700,7 +27700,7 @@ class Problem { }"," } }",113.88770000485238,,Completions generated successfully. Ready for display in your IDE.,2c535526-8b7a-4a44-a29b-b01dfe2495b3,1,deepseek-ai/deepseek-coder-1.3b-base," return sum; -",88,1.0 +",88,1.0 2025-09-22T13:59:00.971455,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27762,7 +27762,7 @@ class Problem { } }",203.65209999727085,,Completions generated successfully. Ready for display in your IDE.,9ef29577-9508-43ff-aa3c-501b89ed9205,1,deepseek-ai/deepseek-coder-1.3b-base," double number = Math.sqrt(item); -",174,0.9942 +",174,0.9942 2025-09-22T13:59:01.059572,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -27805,7 +27805,7 @@ class Problem { } }",88.33040000172332,,Completions generated successfully. Ready for display in your IDE.,da4365a2-e8dd-4535-9d46-9016be029e33,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T13:59:01.230412,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -27838,7 +27838,7 @@ class Problem { ",170.87800000444986,,Completions generated successfully. Ready for display in your IDE.,00c6af75-b0f7-41b9-9fe4-82955c8419d4,1,deepseek-ai/deepseek-coder-1.3b-base," while (i < n - 2): -",146,0.9912 +",146,0.9912 2025-09-22T13:59:01.386085,[1],"//You are given two intervals, // where each interval is a pair of integers. For example, interval = (start, end) = (1, 2). // The given intervals are closed which means that the interval (start, end) @@ -27868,7 +27868,7 @@ function intersection(interval1, interval2){ } ",154.6546000026865,,Completions generated successfully. Ready for display in your IDE.,20ce2dda-3bc3-4388-bfa3-14bc0adb51fc,1,deepseek-ai/deepseek-coder-1.3b-base," if ((end < start)) { -",129,0.957 +",129,0.957 2025-09-22T13:59:01.479483,[1],"from typing import List def match_parens(lst: List[str]) -> str: @@ -27904,7 +27904,7 @@ def match_parens(lst: List[str]) -> str: return 'No' ",93.28249999816762,,Completions generated successfully. Ready for display in your IDE.,b84dc680-f411-4f91-9e8e-9a42b63302e8,1,deepseek-ai/deepseek-coder-1.3b-base," break -",67,1.0 +",67,1.0 2025-09-22T13:59:01.655528,[1],"//Write a function count_nums which takes an array of integers and returns // the number of elements which has a sum of digits > 0. // If a number is negative, then its first signed digit will be negative: @@ -27938,7 +27938,7 @@ function count_nums(arr){ return count; }",176.23530000128085,,Completions generated successfully. Ready for display in your IDE.,910142fc-aaf5-4dee-a7b4-b48cb9f691ea,1,deepseek-ai/deepseek-coder-1.3b-base," sum += Number(str[j]); -",153,1.0 +",153,1.0 2025-09-22T13:59:01.856883,[1],"//brackets is a string of ""<"" and "">"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""<"") @@ -27964,7 +27964,7 @@ function correct_bracketing(brackets){ } ",201.03759999619797,,Completions generated successfully. Ready for display in your IDE.,20e1c5f2-7ee7-4de9-950d-47ad09aba242,1,deepseek-ai/deepseek-coder-1.3b-base," if (brackets[i] === ""<"") { -",176,1.0 +",176,1.0 2025-09-22T13:59:02.055699,[1],"def special_factorial(n: int) -> int: """"""The Brazilian factorial is defined as: brazilian_factorial(n) = n! * (n-1)! * (n-2)! * ... * 1! @@ -27986,7 +27986,7 @@ function correct_bracketing(brackets){ return n * special_factorial(n-1) * factorial(n-1) ",198.5930999944685,,Completions generated successfully. Ready for display in your IDE.,4c830b36-8b9c-48a6-8a52-b90ac41728b6,1,deepseek-ai/deepseek-coder-1.3b-base," - return n * special_factorial(n-1)",175,0.9536 + return n * special_factorial(n-1)",175,0.9536 2025-09-22T13:59:02.201147,[1],"//Given a non-empty array of integers, return the sum of all of the odd elements that are in even positions. // Examples // >>> solution([5, 8, 7, 1]) @@ -28005,7 +28005,7 @@ function solution(lst){ return result; }",145.04569999553496,,Completions generated successfully. Ready for display in your IDE.,18865d8f-d96e-4ac8-a215-710a00771151,1,deepseek-ai/deepseek-coder-1.3b-base," result += lst[i]; -",123,1.0 +",123,1.0 2025-09-22T13:59:02.316818,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -28082,7 +28082,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",115.74829999881331,,Completions generated successfully. Ready for display in your IDE.,766694b0-c85c-480c-9533-7c92958c4383,1,deepseek-ai/deepseek-coder-1.3b-base," return a -",88,1.0 +",88,1.0 2025-09-22T13:59:02.416344,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28126,7 +28126,7 @@ class Problem { } }",98.7950000053388,,Completions generated successfully. Ready for display in your IDE.,f4538971-db2c-4334-bd29-c97356c876be,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:02.570023,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -28181,7 +28181,7 @@ function by_length(arr){ return arr3; }",153.73460000409978,,Completions generated successfully. Ready for display in your IDE.,e6b3e961-82b1-4e6b-93bf-35605d7847ce,1,deepseek-ai/deepseek-coder-1.3b-base," else if(num == 3){ -",130,1.0 +",130,1.0 2025-09-22T13:59:02.748166,[1],"def modp(n: int, p: int) -> int: """"""Return 2^n modulo p (be aware of numerics). >>> modp(3, 5) @@ -28200,7 +28200,7 @@ function by_length(arr){ return 1 if n == 0 else 2**(n % (p-1)) % p ",178.22610000439454,,Completions generated successfully. Ready for display in your IDE.,e25299df-ae29-4cd8-92db-c1b07ef659eb,1,deepseek-ai/deepseek-coder-1.3b-base," raise ValueError(""modp: invalid p"") -",156,1.0 +",156,1.0 2025-09-22T13:59:02.858114,[1],"def largest_prime_factor(n: int) -> int: """"""Return the largest prime factor of n. Assume n > 1 and is not a prime. >>> largest_prime_factor(13195) @@ -28218,7 +28218,7 @@ function by_length(arr){ ",109.40480000135722,,Completions generated successfully. Ready for display in your IDE.,13b6a1a9-9676-4014-87e0-5c26e9708efe,1,deepseek-ai/deepseek-coder-1.3b-base," factor = 2 -",84,1.0 +",84,1.0 2025-09-22T13:59:03.025297,[1],"from typing import List def smallest_change(arr: List[int]) -> int: @@ -28244,7 +28244,7 @@ def smallest_change(arr: List[int]) -> int: return count ",166.6038999974262,,Completions generated successfully. Ready for display in your IDE.,88ac55b1-a85a-481e-bdaa-dcc0e008918e,1,deepseek-ai/deepseek-coder-1.3b-base," end = len(arr) - 1 -",142,1.0 +",142,1.0 2025-09-22T13:59:03.413260,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28306,7 +28306,7 @@ class Problem { } }",387.78959999763174,,Completions generated successfully. Ready for display in your IDE.,b8f3fb6d-bfde-4ec6-a8aa-b3eb1e079420,1,deepseek-ai/deepseek-coder-1.3b-base," while ((int)(Math.pow(10,(double)counter+1)) <= Math.sqrt(number)) { -",341,0.9408 +",341,0.9408 2025-09-22T13:59:03.643938,[1],"//Given array of integers, return array in strange order. // Strange sorting, is when you start with the minimum value, // then maximum of the remaining integers, then minimum and so on. @@ -28334,7 +28334,7 @@ function strange_sort_list(lst){ }",230.28469999553636,,Completions generated successfully. Ready for display in your IDE.,a356197b-2d41-42fc-b67a-6f899f68173e,1,deepseek-ai/deepseek-coder-1.3b-base," //Sort list using Math.min() & Math.max(). -",206,0.9076 +",206,0.9076 2025-09-22T13:59:03.721931,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28375,7 +28375,7 @@ class Problem { return result; } }",78.41850000113482,,Completions generated successfully. Ready for display in your IDE.,3bae6d64-87f9-47e4-9282-902ecae54106,1,deepseek-ai/deepseek-coder-1.3b-base," - ",48,1.0 + ",48,1.0 2025-09-22T13:59:04.162768,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -28425,7 +28425,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",440.2556999993976,,Completions generated successfully. Ready for display in your IDE.,db23f701-0c9b-4cfb-8bb8-b729a7ad9f7d,1,deepseek-ai/deepseek-coder-1.3b-base," if(isPrime(fib(fibNum)) == true && !Number.isInteger(Math.sqrt(fib(fibNum))) ) { -",416,0.9565 +",416,0.9565 2025-09-22T13:59:04.249043,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -28455,7 +28455,7 @@ function parse_nested_parens(paren_string){ ",86.17880000383593,,Completions generated successfully. Ready for display in your IDE.,1a6ca84d-35c0-4f62-889f-520352308ef6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:04.345481,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28494,7 +28494,7 @@ class Problem { } }",95.88459999940824,,Completions generated successfully. Ready for display in your IDE.,c64cf5a1-3545-4b08-a622-5e4fb166f11d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:59:04.688119,[1],"from typing import List def smallest_change(arr: List[int]) -> int: @@ -28521,7 +28521,7 @@ def smallest_change(arr: List[int]) -> int: ",342.58630000113044,,Completions generated successfully. Ready for display in your IDE.,781d8ae9-e4a6-4bec-b329-5cf47377a733,1,deepseek-ai/deepseek-coder-1.3b-base," if arr[start] > arr[end]: count += (arr[start] - arr[end]) + 1 -",319,0.9755 +",319,0.9755 2025-09-22T13:59:04.906737,[1],"from typing import List, Tuple, Optional def largest_smallest_integers(lst: List[int]) -> Tuple[Optional[int], Optional[int]]: @@ -28549,7 +28549,7 @@ def largest_smallest_integers(lst: List[int]) -> Tuple[Optional[int], Optional[i return (largest_negative, smallest_positive) ",218.24480000213953,,Completions generated successfully. Ready for display in your IDE.,3be8a53e-b2c3-4820-9b5e-e760bf68aa02,1,deepseek-ai/deepseek-coder-1.3b-base," if smallest_positive is None or num < smallest_positive: -",193,1.0 +",193,1.0 2025-09-22T13:59:05.044029,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -28578,7 +28578,7 @@ function f(n){ return result; }",136.9838000027812,,Completions generated successfully. Ready for display in your IDE.,574d9b19-d8a1-48bd-8d34-ca3bcb9f86c0,1,deepseek-ai/deepseek-coder-1.3b-base," var factorial = 1; -",113,1.0 +",113,1.0 2025-09-22T13:59:05.739459,[1],"//Create a function that takes a value (string) representing a number // and returns the closest integer to it. If the number is equidistant // from two integers, round it away from zero. @@ -28613,7 +28613,7 @@ function closest_integer(value){ const absValue = Math.abs(value); console.log('Abs Value', absValue); if(!isNaN(absValue)){ -",669,0.8937 +",669,0.8937 2025-09-22T13:59:05.792875,[1],"from typing import List def sort_array(array: List[int]) -> List[int]: @@ -28637,7 +28637,7 @@ def sort_array(array: List[int]) -> List[int]: """""" if array: return sorted(array, reverse=True) if (array[0] + array[-1]) % 2 == 0 else sorted(array)",,53.631299997505266,,Completions generated successfully. Ready for display in your IDE.,fbcc3c37-8704-489f-a4c1-15e10e5ef017,1,deepseek-ai/deepseek-coder-1.3b-base," -",30,0.9246 +",30,0.9246 2025-09-22T13:59:05.865545,[1],"//Filter an input array of strings only for ones that contain given substring // >>> filter_by_substring([], ""a"") // [] @@ -28651,7 +28651,7 @@ function filter_by_substring(strings, substring){ } } return filtered;",,72.29659999575233,,Completions generated successfully. Ready for display in your IDE.,efcfd336-d83a-4642-bb18-13b0d6a5669f,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T13:59:05.960126,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28689,7 +28689,7 @@ class Problem { } return result;",},93.91770000365796,,Completions generated successfully. Ready for display in your IDE.,481ae84f-264f-4592-b7ad-696fec27d03d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:06.046145,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28713,7 +28713,7 @@ class Problem { } }",86.20690000680042,,Completions generated successfully. Ready for display in your IDE.,8ceeb8be-958d-4eb3-8bf3-228172d1c6f8,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:59:06.268946,[1],"//Given array of integers, return array in strange order. // Strange sorting, is when you start with the minimum value, // then maximum of the remaining integers, then minimum and so on. @@ -28740,7 +28740,7 @@ function strange_sort_list(lst){ return output; }",222.25299999990966,,Completions generated successfully. Ready for display in your IDE.,2579581b-e9e4-4f9e-8e74-48de4e3c438d,1,deepseek-ai/deepseek-coder-1.3b-base," output.push(Math.max.apply(null, tmp)); -",198,1.0 +",198,1.0 2025-09-22T13:59:06.562305,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28783,7 +28783,7 @@ class Problem { }",293.2291000033729,,Completions generated successfully. Ready for display in your IDE.,69668df5-c80f-4ac1-ac84-9fa0710b346c,1,deepseek-ai/deepseek-coder-1.3b-base," else { return Integer.compare(s1.length(), s2.length()); } -",266,0.9551 +",266,0.9551 2025-09-22T13:59:06.730195,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -28838,7 +28838,7 @@ function by_length(arr){ return arr3; }",167.68379999848548,,Completions generated successfully. Ready for display in your IDE.,d404ca41-9c60-401e-8733-075976f1df25,1,deepseek-ai/deepseek-coder-1.3b-base," arr3.push(""Six""); -",141,1.0 +",141,1.0 2025-09-22T13:59:06.920986,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28878,7 +28878,7 @@ class Problem { return array; } }",190.5996000059531,,Completions generated successfully. Ready for display in your IDE.,4c684b50-27f7-49d1-adec-f1febd0c88e1,1,deepseek-ai/deepseek-coder-1.3b-base," if(array == null || array.isEmpty()) { -",165,1.0 +",165,1.0 2025-09-22T13:59:07.061609,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -28956,7 +28956,7 @@ class Problem { } }",140.19220000045607,,Completions generated successfully. Ready for display in your IDE.,23f1a188-4d24-41d4-a7ed-6eda6d17eea5,1,deepseek-ai/deepseek-coder-1.3b-base," res += ""xc""; -",113,1.0 +",113,1.0 2025-09-22T13:59:07.170411,[1],"//Your task is to write a function that returns true if a number x is a simple // power of n and false in other cases. // x is a simple power of n if n**int=x @@ -28985,7 +28985,7 @@ function is_simple_power(x, n){ } ",108.81479999807198,,Completions generated successfully. Ready for display in your IDE.,c2bfef67-9e1d-42b4-b0d9-61e40738cec3,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",84,1.0 +",84,1.0 2025-09-22T13:59:07.348220,[1],"//Given a non-empty array of integers arr and an integer k, return // the sum of the elements with at most two digits from the first k elements of arr. // Example: @@ -29005,7 +29005,7 @@ function add_elements(arr, k){ break; } } -",155,0.9946 +",155,0.9946 2025-09-22T13:59:07.499496,[1],"//You are given a string representing a sentence, // the sentence contains some words separated by a space, // and you have to return a string that contains the words from the original sentence, @@ -29035,7 +29035,7 @@ function words_in_sentence(sentence){ }).join(' '); }",150.75390000129119,,Completions generated successfully. Ready for display in your IDE.,cb32a51f-2643-4cc2-bdcc-8d8a3ca6eea0,1,deepseek-ai/deepseek-coder-1.3b-base," let isPrime = true; -",125,1.0 +",125,1.0 2025-09-22T13:59:07.658436,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -29093,7 +29093,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",159.1504999960307,,Completions generated successfully. Ready for display in your IDE.,fd487a76-1484-47fa-9381-2fcb0900a3a9,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('A-') -",132,1.0 +",132,1.0 2025-09-22T13:59:07.808380,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29125,7 +29125,7 @@ class Problem { } }",149.69589999964228,,Completions generated successfully. Ready for display in your IDE.,b4c9b42d-2a3b-4a47-8d84-cec1f006694d,1,deepseek-ai/deepseek-coder-1.3b-base," current_depth -= 1; -",126,1.0 +",126,1.0 2025-09-22T13:59:08.353551,[1],"//Circular shift the digits of the integer x, shift the digits right by shift // and return the result as a string. // If shift > number of digits, return digits reversed. @@ -29148,7 +29148,7 @@ function circular_shift(x, shift){ //if shift is greater than length of input, reverse it str = str.substring(len - shift); res = str + str.substring(0, len - shift); -",505,0.8737 +",505,0.8737 2025-09-22T13:59:08.643933,[1],"from typing import List def sort_even(l: List[int]) -> List[int]: @@ -29168,7 +29168,7 @@ def sort_even(l: List[int]) -> List[int]: return l",290.4731000016909,,Completions generated successfully. Ready for display in your IDE.,e36bd28d-4c40-4bed-ac55-47ee7b68b249,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(l)): if not (i % 2): - ",267,0.9983 + ",267,0.9983 2025-09-22T13:59:08.777317,[1],"from typing import List def find_max(words: List[str]) -> str: @@ -29195,7 +29195,7 @@ def find_max(words: List[str]) -> str: return max_word ",133.04649999918183,,Completions generated successfully. Ready for display in your IDE.,4106d775-5cc1-406d-b915-e18ddf7f9807,1,deepseek-ai/deepseek-coder-1.3b-base," for word in words: -",105,1.0 +",105,1.0 2025-09-22T13:59:08.863598,[1],"//Write a function that takes an array of numbers as input and returns // the number of elements in the array that are greater than 10 and both // first and last digits of a number are odd (1, 3, 5, 7, 9). @@ -29216,7 +29216,7 @@ function specialFilter(nums){ } ",86.2894999954733,,Completions generated successfully. Ready for display in your IDE.,825085a5-1583-4fd9-b62a-b2d09fbd46b5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,0.9242 +",62,0.9242 2025-09-22T13:59:09.127156,[1],"from typing import List def Strongest_Extension(class_name: str, extensions: List[str]) -> str: @@ -29244,7 +29244,7 @@ def Strongest_Extension(class_name: str, extensions: List[str]) -> str: best_extension = extension return class_name + '.' + best_extension",262.91240000136895,,Completions generated successfully. Ready for display in your IDE.,af4f7015-0f2b-4d89-8d7b-1588bed48f61,1,deepseek-ai/deepseek-coder-1.3b-base," if n_uppercase - n_lowercase > best_strength: -",235,1.0 +",235,1.0 2025-09-22T13:59:09.266621,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29294,7 +29294,7 @@ class Problem { } }",139.5436000020709,,Completions generated successfully. Ready for display in your IDE.,dce0fc7b-3084-4655-b7dc-6ccd033475df,1,deepseek-ai/deepseek-coder-1.3b-base," switch (c) { -",111,1.0 +",111,1.0 2025-09-22T13:59:09.537488,[1],"def count_upper(s: str) -> int: """""" Given a string s, count the number of uppercase vowels in even indices. @@ -29318,7 +29318,7 @@ D.1 Recursion Given a positive integer n, write a recursive function that returns the sum of the squares of the positive odd integers smaller than n. Assume n is greater than or equal to 3. """"""",270.51139999821316,,Completions generated successfully. Ready for display in your IDE.,aa2bf01a-3561-4f02-8909-c4ec11458547,1,deepseek-ai/deepseek-coder-1.3b-base," vowel_set = {'a', 'e', 'i', 'o', 'u'} -",246,0.9734 +",246,0.9734 2025-09-22T13:59:09.593180,[1],"//Input to this function is a string containing multiple groups of nested parentheses. Your goal is to // separate those group into separate strings and return the array of those. // Separate groups are balanced (each open brace is properly closed) and not nested within each other @@ -29343,7 +29343,7 @@ function separate_paren_groups(paren_string){ return groups; } ",56.05549999745563,,Completions generated successfully. Ready for display in your IDE.,5ac318aa-cb1b-468e-9b49-8cb695289ca5,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:59:09.749284,[1],"from typing import List def parse_nested_parens(paren_string: str) -> List[int]: @@ -29366,7 +29366,7 @@ def parse_nested_parens(paren_string: str) -> List[int]: return result ",156.00850000191713,,Completions generated successfully. Ready for display in your IDE.,be15f421-5ab8-4411-9d20-cb530a0c11fd,1,deepseek-ai/deepseek-coder-1.3b-base," for paren in parens: -",132,1.0 +",132,1.0 2025-09-22T13:59:09.830206,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -29437,7 +29437,7 @@ function minPath(grid, k){ } return path; }",80.51709999563172,,Completions generated successfully. Ready for display in your IDE.,34a1eafb-e5c8-4870-995a-f27ce7bb5155,1,deepseek-ai/deepseek-coder-1.3b-base," -",52,1.0 +",52,1.0 2025-09-22T13:59:09.928518,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -29468,7 +29468,7 @@ function same_chars(s0, s1){ ",97.51069999765605,,Completions generated successfully. Ready for display in your IDE.,1e003a7b-64bf-4570-8d6d-d3fee9d0e962,1,deepseek-ai/deepseek-coder-1.3b-base," break; -",73,1.0 +",73,1.0 2025-09-22T13:59:10.627404,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -29514,7 +29514,7 @@ function compare_one(a, b){ if ((isNaN(parseFloat(a)) || !isFinite(a))) { console.log('Not an integer'); - }else if((isNaN(parseFloat(b)) || !isFinite(b)))",674,0.9511 + }else if((isNaN(parseFloat(b)) || !isFinite(b)))",674,0.9511 2025-09-22T13:59:10.744871,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29542,7 +29542,7 @@ class Problem { } }",117.42789999698289,,Completions generated successfully. Ready for display in your IDE.,1e378503-10d1-4e29-93f5-6e95168490b1,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1; -",93,1.0 +",93,1.0 2025-09-22T13:59:10.886233,[1],"def encode(message: str) -> str: """""" Write a function that takes a message, and encodes in such a @@ -29569,7 +29569,7 @@ class Problem { return ''.join(map(lambda ch: encode_vowel(swap_case(ch)), message)) ",141.21639999939362,,Completions generated successfully. Ready for display in your IDE.,6c1fea92-9d80-449d-a0e5-5b546bd56c10,1,deepseek-ai/deepseek-coder-1.3b-base," if ch.isalpha(): -",114,0.9775 +",114,0.9775 2025-09-22T13:59:10.978464,[1],"from typing import List def specialFilter(nums: List[int]) -> int: @@ -29590,7 +29590,7 @@ def specialFilter(nums: List[int]) -> int: rev = int(str(num)[::-1]) if num % 2 != 0 and rev % 2 != 0: count += 1",,92.041000003519,,Completions generated successfully. Ready for display in your IDE.,9bea6dc5-f7a5-4aa6-ba06-f9f21bf61f0d,1,deepseek-ai/deepseek-coder-1.3b-base," - return count",62,0.9596 + return count",62,0.9596 2025-09-22T13:59:11.063929,[1],"//brackets is a string of ""<"" and "">"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""<"") @@ -29616,7 +29616,7 @@ function correct_bracketing(brackets){ } ",84.55939999839757,,Completions generated successfully. Ready for display in your IDE.,e24043dd-653a-4027-abd9-ec047b3f9f22,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:11.150077,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29637,7 +29637,7 @@ class Problem { } return result.trim();",},86.30280000215862,,Completions generated successfully. Ready for display in your IDE.,d7476345-5d50-405c-b8e6-dd11fd874c7a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:59:11.330466,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29666,7 +29666,7 @@ class Problem { return primes.get(primes.size() - 1); } }",180.30719999660505,,Completions generated successfully. Ready for display in your IDE.,882baf26-2f5f-48e8-a762-f91b206b4049,1,deepseek-ai/deepseek-coder-1.3b-base," List primes = new ArrayList<>(); -",156,1.0 +",156,1.0 2025-09-22T13:59:11.402163,[1],"//You are given an array of integers. // You need to find the largest prime value and return the sum of its digits. // Examples: @@ -29697,7 +29697,7 @@ function skjkasdkd(lst){ sum += Number(digits[i]); } return sum;",,71.78340000245953,,Completions generated successfully. Ready for display in your IDE.,7af4e63c-6afe-4f8d-97bf-36abfeab24e1,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T13:59:11.444581,[1],"//Return median of elements in the array l. // >>> median([3, 1, 2, 4, 5]) // 3 @@ -29711,7 +29711,7 @@ function median(l){ return sorted[middle]; } } -",41.493499993521255,,Completions generated successfully. Ready for display in your IDE.,75d53283-9ae8-48b2-88bf-db9252ee1b3a,1,deepseek-ai/deepseek-coder-1.3b-base,,20,0.5919 +",41.493499993521255,,Completions generated successfully. Ready for display in your IDE.,75d53283-9ae8-48b2-88bf-db9252ee1b3a,1,deepseek-ai/deepseek-coder-1.3b-base,,20,0.5919 2025-09-22T13:59:11.726684,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29742,7 +29742,7 @@ class Problem { }",282.19639999588253,,Completions generated successfully. Ready for display in your IDE.,9e996d84-a979-4755-9bb9-47f09b82ca7e,1,deepseek-ai/deepseek-coder-1.3b-base," char c = s.charAt(j); if (c == '7') { -",259,1.0 +",259,1.0 2025-09-22T13:59:11.862054,[1],"from typing import List def get_positive(l: List[int]) -> List[int]: @@ -29757,7 +29757,7 @@ def get_positive(l: List[int]) -> List[int]: if i > 0:"," return p ",134.9019999979646,,Completions generated successfully. Ready for display in your IDE.,3f96db7a-b4e2-4bfc-951b-fd4e32aae8ed,1,deepseek-ai/deepseek-coder-1.3b-base," p.append(i) -",113,1.0 +",113,1.0 2025-09-22T13:59:11.918123,[1],"def flip_case(string: str) -> str: """""" For a given string, flip lowercase characters to uppercase and uppercase to lowercase. >>> flip_case('Hello') @@ -29770,7 +29770,7 @@ def get_positive(l: List[int]) -> List[int]: new_string += character.upper() return new_string -",55.99709999660263,,Completions generated successfully. Ready for display in your IDE.,54bdd415-e946-4fae-bfeb-a063d590c74a,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 +",55.99709999660263,,Completions generated successfully. Ready for display in your IDE.,54bdd415-e946-4fae-bfeb-a063d590c74a,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 2025-09-22T13:59:12.051432,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29808,7 +29808,7 @@ class Problem { } }",133.30409999616677,,Completions generated successfully. Ready for display in your IDE.,649609a8-ccb1-4233-9bb9-78587c129ca7,1,deepseek-ai/deepseek-coder-1.3b-base," int min = 0; -",109,1.0 +",109,1.0 2025-09-22T13:59:12.142133,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29841,7 +29841,7 @@ class Problem { } }",89.93499999633059,,Completions generated successfully. Ready for display in your IDE.,9123dc2f-b803-494f-82d6-f76302ace5a1,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:59:12.226953,[1],"//Given an array of positive integers x. return a sorted array of all // elements that hasn't any even digit. // Note: Returned array should be sorted in increasing order. @@ -29861,7 +29861,7 @@ function unique_digits(x){ ",85.10259999457048,,Completions generated successfully. Ready for display in your IDE.,0f6ce1cd-4d29-4d63-94ca-cafb9cb244df,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:12.383786,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -29916,7 +29916,7 @@ function by_length(arr){ } }",},157.01419999822974,,Completions generated successfully. Ready for display in your IDE.,9519f486-6df7-43b7-979e-dfafd9cf97dd,1,deepseek-ai/deepseek-coder-1.3b-base," console.log(arr3); -",130,1.0 +",130,1.0 2025-09-22T13:59:12.491327,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29946,7 +29946,7 @@ class Problem { }"," } }",106.57940000237431,,Completions generated successfully. Ready for display in your IDE.,c9a1ac0e-e834-4a17-a96f-cf0ec3033701,1,deepseek-ai/deepseek-coder-1.3b-base," return res; -",82,1.0 +",82,1.0 2025-09-22T13:59:12.669431,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -29994,7 +29994,7 @@ class Problem { } }",178.57749999529915,,Completions generated successfully. Ready for display in your IDE.,338b769f-ea3b-44e0-811f-50de4e8dc937,1,deepseek-ai/deepseek-coder-1.3b-base," indexMax = indexPlanet1; -",149,1.0 +",149,1.0 2025-09-22T13:59:12.730823,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30022,7 +30022,7 @@ class Problem { if (n == 2) return 2;"," return fib4(n-1) + fib4(n-2) + fib4(n-3) + fib4(n-4); } }",60.838900004455354,,Completions generated successfully. Ready for display in your IDE.,8971f7b0-680e-47d2-ae51-99427e0a2be9,1,deepseek-ai/deepseek-coder-1.3b-base," -",33,1.0 +",33,1.0 2025-09-22T13:59:12.992532,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30055,7 +30055,7 @@ class Problem { }",261.18549999955576,,Completions generated successfully. Ready for display in your IDE.,78794d56-97d8-4cbb-9204-4cab5b584886,1,deepseek-ai/deepseek-coder-1.3b-base," while ((left < right) && (isPalindrome == true)) { -",238,0.9772 +",238,0.9772 2025-09-22T13:59:13.055959,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30075,7 +30075,7 @@ class Problem { } return 1; }",,63.48189999698661,,Completions generated successfully. Ready for display in your IDE.,70ad8bf8-57f4-439f-9ef5-13aac85a87b6,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:59:13.329689,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30128,7 +30128,7 @@ class Problem { } }",273.6475999990944,,Completions generated successfully. Ready for display in your IDE.,c93c9780-ff80-46f5-b1fd-380ead1e5048,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < arr.size(); ++i) { -",247,1.0 +",247,1.0 2025-09-22T13:59:13.539080,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30154,7 +30154,7 @@ class Problem { } }",208.9759000009508,,Completions generated successfully. Ready for display in your IDE.,9d7b9783-3fb0-48bc-97de-b91913493081,1,deepseek-ai/deepseek-coder-1.3b-base," long y = Math.abs(b % 10); -",186,0.9578 +",186,0.9578 2025-09-22T13:59:13.704532,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30194,7 +30194,7 @@ class Problem { } }",165.5077999967034,,Completions generated successfully. Ready for display in your IDE.,63d91502-218a-4050-a279-240cedb91851,1,deepseek-ai/deepseek-coder-1.3b-base," if (compareResult == 0) { -",140,1.0 +",140,1.0 2025-09-22T13:59:13.808004,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30233,7 +30233,7 @@ class Problem { } }",103.22879999876022,,Completions generated successfully. Ready for display in your IDE.,e75bb758-47a8-4d0b-97b4-c68af9d7007d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",76,1.0 +",76,1.0 2025-09-22T13:59:13.901236,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -30259,7 +30259,7 @@ function string_xor(a, b){ return result; }",93.33769999648212,,Completions generated successfully. Ready for display in your IDE.,6ba7bfd8-eb02-447b-8354-5a933109694f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:13.956451,[1],"def count_upper(s: str) -> int: """""" Given a string s, count the number of uppercase vowels in even indices. @@ -30281,7 +30281,7 @@ function string_xor(a, b){ """"""","Given a positive integer n, write a recursive function that returns the sum of the squares of the positive odd integers smaller than n. Assume n is greater than or equal to 3. -""""""",54.90109999664128,,Completions generated successfully. Ready for display in your IDE.,33c94c3d-32c0-4068-865b-79f92c6cfc52,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +""""""",54.90109999664128,,Completions generated successfully. Ready for display in your IDE.,33c94c3d-32c0-4068-865b-79f92c6cfc52,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:59:14.077587,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30343,7 +30343,7 @@ class Problem { } }",120.36800000350922,,Completions generated successfully. Ready for display in your IDE.,ecda7b8d-16bb-43df-9c23-4abfa2920414,1,deepseek-ai/deepseek-coder-1.3b-base," } -",91,1.0 +",91,1.0 2025-09-22T13:59:14.151835,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -30376,7 +30376,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",74.04449999739882,,Completions generated successfully. Ready for display in your IDE.,83eb826d-a6cf-4f6b-ac6d-17ab38db9807,1,deepseek-ai/deepseek-coder-1.3b-base," - ",48,0.9689 + ",48,0.9689 2025-09-22T13:59:14.332642,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30401,7 +30401,7 @@ class Problem { } }",181.07400000008056,,Completions generated successfully. Ready for display in your IDE.,fa5251b1-c384-4331-9877-c7d9fdd8cfd6,1,deepseek-ai/deepseek-coder-1.3b-base," char c = string.charAt(i); -",155,1.0 +",155,1.0 2025-09-22T13:59:14.488086,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30434,7 +30434,7 @@ class Problem { } }",154.9765000017942,,Completions generated successfully. Ready for display in your IDE.,501fcdda-9a62-48b0-b0b4-f0ffc929549d,1,deepseek-ai/deepseek-coder-1.3b-base," v *= j; -",105,1.0 +",105,1.0 2025-09-22T13:59:14.574689,[1],"from typing import Tuple def even_odd_palindrome(n: int) -> Tuple[int, int]: @@ -30463,7 +30463,7 @@ def even_odd_palindrome(n: int) -> Tuple[int, int]: return ( sum(1 for i in range(1, n + 1) if str(i) == str(i)[::-1] and i % 2 == 0), sum(1 for i in range(1, n + 1) if str(i) == str(i)[::-1] and i % 2 != 0)",,85.80870000150753,,Completions generated successfully. Ready for display in your IDE.,e7b93de3-b5ce-4dc2-9780-cacbf09cf985,1,deepseek-ai/deepseek-coder-1.3b-base,", - )",62,0.8622 + )",62,0.8622 2025-09-22T13:59:14.653072,[1],"//Evaluate whether the given number n can be written as the sum of exactly 4 positive even numbers // Example // >>> is_equal_to_sum_even(4) @@ -30492,7 +30492,7 @@ function is_equal_to_sum_even(n){ } }",78.3785000021453,,Completions generated successfully. Ready for display in your IDE.,300e0e63-48f0-4bfc-a339-5a8d7e430589,1,deepseek-ai/deepseek-coder-1.3b-base," } -",56,1.0 +",56,1.0 2025-09-22T13:59:14.771963,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30539,7 +30539,7 @@ class Problem { } return count;",},118.89059999521123,,Completions generated successfully. Ready for display in your IDE.,d6c6e3b7-1704-40fa-a147-3e27fe039165,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:59:15.180581,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30579,7 +30579,7 @@ class Problem { } }",408.4459999940009,,Completions generated successfully. Ready for display in your IDE.,4da389d2-32a2-4402-aece-9dc4c02029a4,1,deepseek-ai/deepseek-coder-1.3b-base," int compareResult = wordsToUniqueCharacters.get(o1).compareTo(wordsToUniqueCharacters.get(o2)); -",378,0.9857 +",378,0.9857 2025-09-22T13:59:15.324464,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30617,7 +30617,7 @@ class Problem { } }",143.86260000173934,,Completions generated successfully. Ready for display in your IDE.,ab2e7d4f-8bc7-4d28-8ea2-9b36302a5b55,1,deepseek-ai/deepseek-coder-1.3b-base," return lst2; -",117,1.0 +",117,1.0 2025-09-22T13:59:15.439266,[1],"//This function takes two positive numbers x and y and returns the // biggest even integer number that is in the range [x, y] inclusive. If // there's no such number, then the function should return -1. @@ -30635,7 +30635,7 @@ function choose_num(x, y){ return num; } ",114.7722999958205,,Completions generated successfully. Ready for display in your IDE.,3fe2f851-cfe3-4dad-8b47-cc159a5a3c51,1,deepseek-ai/deepseek-coder-1.3b-base," var num = 0; -",87,1.0 +",87,1.0 2025-09-22T13:59:15.570456,[1],"def int_to_mini_roman(number: int) -> str: """""" Given a positive integer, obtain its roman numeral equivalent as a string, @@ -30671,7 +30671,7 @@ function choose_num(x, y){ ",130.5893999960972,,Completions generated successfully. Ready for display in your IDE.,f81a1983-6d35-425d-9819-5594766c07b0,1,deepseek-ai/deepseek-coder-1.3b-base," 1: 'i', -",105,0.9159 +",105,0.9159 2025-09-22T13:59:15.696591,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30703,7 +30703,7 @@ class Problem { } }",125.904200001969,,Completions generated successfully. Ready for display in your IDE.,1068b0f6-d6af-4d80-90cb-300cfb3c9f4e,1,deepseek-ai/deepseek-coder-1.3b-base," int left = 0; -",102,1.0 +",102,1.0 2025-09-22T13:59:15.793770,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30748,7 +30748,7 @@ class Problem { } return true;",},96.80040000239387,,Completions generated successfully. Ready for display in your IDE.,b3f4ad7b-d077-423a-b9e9-d2ad5a82a8fe,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:59:15.944315,[1],"from typing import List def find_max(words: List[str]) -> str: @@ -30775,7 +30775,7 @@ def find_max(words: List[str]) -> str: return max_word ",150.68959999916842,,Completions generated successfully. Ready for display in your IDE.,a86d7ebb-6e2c-48bc-944c-29ffedb85f43,1,deepseek-ai/deepseek-coder-1.3b-base," if word < max_word: -",124,1.0 +",124,1.0 2025-09-22T13:59:16.039437,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30811,7 +30811,7 @@ class Problem { } }",94.52310000051511,,Completions generated successfully. Ready for display in your IDE.,8f1b643a-659c-4cc6-8b37-0d0f48289360,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:16.157702,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30842,7 +30842,7 @@ class Problem { } }",118.4685999978683,,Completions generated successfully. Ready for display in your IDE.,9964c897-1407-4f99-a7cc-0c8103535281,1,deepseek-ai/deepseek-coder-1.3b-base," inc = false; -",95,1.0 +",95,1.0 2025-09-22T13:59:16.297410,[1],"//Filter given array of any jsthon values only for integers // >>> filter_integers([""a"", 3.14, 5]) // [5] @@ -30857,7 +30857,7 @@ function filter_integers(values){ } ",139.72389999980805,,Completions generated successfully. Ready for display in your IDE.,d6049fd0-b706-4416-8bed-e046d0d1cf16,1,deepseek-ai/deepseek-coder-1.3b-base," result.push(value); -",118,1.0 +",118,1.0 2025-09-22T13:59:16.427611,[1],"//pairs_sum_to_zero takes an array of integers as an input. // it returns true if there are two distinct elements in the array that // sum to zero, and false otherwise. @@ -30885,7 +30885,7 @@ function pairs_sum_to_zero(l){ } ",129.76539999363013,,Completions generated successfully. Ready for display in your IDE.,7051dc52-0051-4e50-bd45-343341a39f86,1,deepseek-ai/deepseek-coder-1.3b-base," i = i + 1 -",104,1.0 +",104,1.0 2025-09-22T13:59:16.570713,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -30923,7 +30923,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",142.86020000145072,,Completions generated successfully. Ready for display in your IDE.,0a333ea4-632d-4ca7-8eaa-409d949f2b31,1,deepseek-ai/deepseek-coder-1.3b-base," return (y == x); -",119,0.8687 +",119,0.8687 2025-09-22T13:59:16.657091,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -30963,7 +30963,7 @@ class Problem { } }",86.6939999978058,,Completions generated successfully. Ready for display in your IDE.,6aae5e67-5045-4c3e-b754-2122c4c9ca0f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:59:16.815204,[1],"//You are given a string s. // if s[i] is a letter, reverse its case from lower to upper or vise versa, // otherwise keep it as it is. @@ -30991,7 +30991,7 @@ function solve(s){ } ",157.0584000000963,,Completions generated successfully. Ready for display in your IDE.,a1f3d4d8-6e5c-42f3-8d87-7397b73429c4,1,deepseek-ai/deepseek-coder-1.3b-base," : char.toLowerCase(); -",124,1.0 +",124,1.0 2025-09-22T13:59:16.962370,[1],"def fizz_buzz(n: int) -> int: """"""Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. >>> fizz_buzz(50) @@ -31006,7 +31006,7 @@ function solve(s){ return x ",146.7720000000554,,Completions generated successfully. Ready for display in your IDE.,e9b4a184-6198-4721-b96b-2e299d82c453,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(n): -",123,1.0 +",123,1.0 2025-09-22T13:59:17.088399,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -31031,7 +31031,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: return groups ",126.1681000032695,,Completions generated successfully. Ready for display in your IDE.,cafb5833-5744-4f85-b4f4-58c4be7053e0,1,deepseek-ai/deepseek-coder-1.3b-base," open_count = 0 -",104,1.0 +",104,1.0 2025-09-22T13:59:17.153651,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31063,7 +31063,7 @@ class Problem { } return sb.reverse().toString(); }",,64.98899999860441,,Completions generated successfully. Ready for display in your IDE.,aee0d838-aead-452b-b3d5-7ae2d8bd4809,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:59:17.258739,[1],"//Given an array of non-negative integers, return a cojs of the given array after sorting, // you will sort the given array in ascending order if the sum( first index value, last index value) is odd, // or sort it in descending order if the sum( first index value, last index value) is even. @@ -31087,7 +31087,7 @@ function sort_array(array){ } return newArr; }",104.98489999736194,,Completions generated successfully. Ready for display in your IDE.,b9af97a5-5b08-422e-acd7-b1a60dfad941,1,deepseek-ai/deepseek-coder-1.3b-base," // your code here -",82,0.9177 +",82,0.9177 2025-09-22T13:59:17.518641,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -31142,7 +31142,7 @@ function by_length(arr){ return arr3; }",259.53600000502774,,Completions generated successfully. Ready for display in your IDE.,5f8c55a7-4a9e-4c67-8a6f-48a4da4b4f16,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=0 ; i 0l) { -",184,0.9384 +",184,0.9384 2025-09-22T13:59:17.842939,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -31261,7 +31261,7 @@ function minPath(grid, k){ return path; }",112.24770000262652,,Completions generated successfully. Ready for display in your IDE.,ddafaca3-ebb2-4c79-a5c6-e999ff4b13ec,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:59:18.247599,[1],"//Create a function encrypt that takes a string as an argument and // returns a string encrypted with the alphabet being rotated. // The alphabet should be rotated in a manner such that the letters @@ -31286,7 +31286,7 @@ function encrypt(s){ ",405.04840000357945,,Completions generated successfully. Ready for display in your IDE.,efc7b107-a18b-4481-9bd7-0418493f1eb1,1,deepseek-ai/deepseek-coder-1.3b-base," //get index of letter from inputted word, then add it to our shifted value var index = (alphabet.indexOf)(s[i]); -",382,0.9175 +",382,0.9175 2025-09-22T13:59:18.839108,[1],"from typing import List def unique_digits(x: List[int]) -> List[int]: @@ -31317,7 +31317,7 @@ def unique_digits(x: List[int]) -> List[int]: for i in range(len(s)): for j in range(len(x)-1): if (s[i]==x[j+1]-x[j]): - ",567,0.9635 + ",567,0.9635 2025-09-22T13:59:18.902784,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -31350,7 +31350,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",63.91090000397526,,Completions generated successfully. Ready for display in your IDE.,529315d1-c2c5-43ea-a6da-32f89009994b,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,0.9621 +",38,0.9621 2025-09-22T13:59:18.957842,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31376,7 +31376,7 @@ class Problem { } } }",54.521899997780565,,Completions generated successfully. Ready for display in your IDE.,4a9cda01-323b-4ce8-b46f-c2eadfdae6cb,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,0.8716 +",31,0.8716 2025-09-22T13:59:19.063386,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31399,7 +31399,7 @@ class Problem { }"," } }",105.97119999874849,,Completions generated successfully. Ready for display in your IDE.,0554ca73-da1e-400a-aa41-8e221d134d70,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",82,0.9796 +",82,0.9796 2025-09-22T13:59:19.349619,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -31449,7 +31449,7 @@ function prime_fib(n){ return fib(fibNum - 1); }",285.2721999952337,,Completions generated successfully. Ready for display in your IDE.,320caf3c-ede8-4b60-abc0-60dd90eaf2d2,1,deepseek-ai/deepseek-coder-1.3b-base," for(var i=5; ((i * i) <= number); i+= 6){ -",260,0.9263 +",260,0.9263 2025-09-22T13:59:19.536803,[1],"//You have been tasked to write a function that receives // a hexadecimal number as a string and counts the number of hexadecimal // digits that are primes (prime number, or a prime, is a natural number @@ -31483,7 +31483,7 @@ function hex_key(num){ continue; } } -",161,1.0 +",161,1.0 2025-09-22T13:59:19.732847,[1],"//Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. // >>> fizz_buzz(50) // 0 @@ -31505,7 +31505,7 @@ function fizz_buzz(n){ } ",195.75560000521364,,Completions generated successfully. Ready for display in your IDE.,c0b9927a-5b0d-4e6d-838d-de07e0bf0388,1,deepseek-ai/deepseek-coder-1.3b-base," if (arr[j] === '7'){ -",173,1.0 +",173,1.0 2025-09-22T13:59:19.825118,[1],"//Input to this function is a string containing multiple groups of nested parentheses. Your goal is to // separate those group into separate strings and return the array of those. // Separate groups are balanced (each open brace is properly closed) and not nested within each other @@ -31531,7 +31531,7 @@ function separate_paren_groups(paren_string){ } ",92.09130000090227,,Completions generated successfully. Ready for display in your IDE.,d714fe59-2da3-4a0f-a9c5-4817352765d8,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,0.9674 +",69,0.9674 2025-09-22T13:59:19.945344,[1],"//You are given a word. Your task is to find the closest vowel that stands between // two consonants from the right side of the word (case sensitive). // Vowels in the beginning and ending doesn't count. Return empty string if you didn't @@ -31569,7 +31569,7 @@ function get_closest_vowel(word){ } }",119.49559999629855,,Completions generated successfully. Ready for display in your IDE.,e2664ff3-a366-4344-8b42-f7a85ddc7188,1,deepseek-ai/deepseek-coder-1.3b-base," for (;;) { -",95,0.8687 +",95,0.8687 2025-09-22T13:59:20.009187,[1],"//Return true is array elements are monotonically increasing or decreasing. // >>> monotonic([1, 2, 4, 20]) // true @@ -31596,7 +31596,7 @@ function monotonic(l){ } } return true;",,64.1990000003716,,Completions generated successfully. Ready for display in your IDE.,a43d600b-b84f-414d-a745-c21fc9965f63,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T13:59:20.707546,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31624,7 +31624,7 @@ class Problem { return al; } - //",674,0.9264 + //",674,0.9264 2025-09-22T13:59:20.921133,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31673,7 +31673,7 @@ class Problem { } }",212.96030000667088,,Completions generated successfully. Ready for display in your IDE.,7dc873d4-8109-4129-b60d-44e91221eb38,1,deepseek-ai/deepseek-coder-1.3b-base," if (!hm.containsKey(sss)) { -",184,0.9762 +",184,0.9762 2025-09-22T13:59:21.028860,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -31699,7 +31699,7 @@ function is_multiply_prime(a){ } }",107.78170000412501,,Completions generated successfully. Ready for display in your IDE.,2db84e94-de80-486b-b8a0-46841c93775f,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",82,1.0 +",82,1.0 2025-09-22T13:59:21.146469,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -31738,7 +31738,7 @@ function valid_date(date){ } ",117.530399998941,,Completions generated successfully. Ready for display in your IDE.,88e52f67-ddf0-427f-b33e-640e133324eb,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",90,0.9124 +",90,0.9124 2025-09-22T13:59:21.359995,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -31768,7 +31768,7 @@ function parse_nested_parens(paren_string){ ",213.30110000417335,,Completions generated successfully. Ready for display in your IDE.,c4783c4e-2167-4d75-85e6-73b89346ad59,1,deepseek-ai/deepseek-coder-1.3b-base," if (paren_string[i] == ""(""){ -",189,1.0 +",189,1.0 2025-09-22T13:59:21.458211,[1],"def make_palindrome(string: str) -> str: """""" Find the shortest palindrome that begins with a supplied string. Algorithm idea is simple: @@ -31791,7 +31791,7 @@ function parse_nested_parens(paren_string){ return string ",97.61799999978393,,Completions generated successfully. Ready for display in your IDE.,ec858b00-f86b-4fd4-b39b-28af6451ef3b,1,deepseek-ai/deepseek-coder-1.3b-base," - prefix = string",73,1.0 + prefix = string",73,1.0 2025-09-22T13:59:21.598809,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31869,7 +31869,7 @@ class Problem { } }",140.72680000390392,,Completions generated successfully. Ready for display in your IDE.,3ecaa910-c323-4a6c-949c-2fc366518677,1,deepseek-ai/deepseek-coder-1.3b-base," num -= 9; -",113,1.0 +",113,1.0 2025-09-22T13:59:21.843707,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31919,7 +31919,7 @@ class Problem { } }",244.59550000028685,,Completions generated successfully. Ready for display in your IDE.,69942065-7cd3-47d1-b3f9-c06387d1862a,1,deepseek-ai/deepseek-coder-1.3b-base," for (char c : wordInLowerCase.toCharArray()) { -",214,1.0 +",214,1.0 2025-09-22T13:59:22.035574,[1],"def sum_to_n(n: int) -> int: """"""sum_to_n is a function that sums numbers from 1 to n. >>> sum_to_n(30) @@ -31935,7 +31935,7 @@ class Problem { """""" "," ",191.45539999590255,,Completions generated successfully. Ready for display in your IDE.,2b16cf60-bafb-430a-b7af-cb0eae52c3d2,1,deepseek-ai/deepseek-coder-1.3b-base," return (n * (n + 1)) // 2 -",169,0.9677 +",169,0.9677 2025-09-22T13:59:22.102278,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -31965,7 +31965,7 @@ class Problem { factors.add(n); return factors; }",,66.8218999999226,,Completions generated successfully. Ready for display in your IDE.,46645aa1-4d40-4a99-8958-205a34a4f272,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T13:59:22.210263,[1],"from typing import List def smallest_change(arr: List[int]) -> int: @@ -31991,7 +31991,7 @@ def smallest_change(arr: List[int]) -> int: return count ",107.24760000448441,,Completions generated successfully. Ready for display in your IDE.,c9c83a4c-5bab-4a4b-9ec9-8f65d238f695,1,deepseek-ai/deepseek-coder-1.3b-base," count = 0 -",83,1.0 +",83,1.0 2025-09-22T13:59:22.793620,[1],"//You will be given a string of words separated by commas or spaces. Your task is // to split the string into words and return an array of the words. // For example: @@ -32003,7 +32003,7 @@ function words_string(s){ "," return res; }",583.2049000018742,,Completions generated successfully. Ready for display in your IDE.,d93415af-1c00-4c4d-bcf3-7ecff1086563,1,deepseek-ai/deepseek-coder-1.3b-base," let arr = s.split(/[ ,]+/); //split() method splits up strings based on regex pattern passed as argument (in this case it'll match any whitespace) console.log('arr', arr); -",558,0.9285 +",558,0.9285 2025-09-22T13:59:23.050038,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32057,7 +32057,7 @@ class Problem { }",255.88620000053197,,Completions generated successfully. Ready for display in your IDE.,e4a63b34-6a7b-474a-bb2a-d34f7924e30d,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList newArr = new ArrayList<>(); -",215,0.993 +",215,0.993 2025-09-22T13:59:23.112190,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32088,7 +32088,7 @@ class Problem { return sum; } }",62.67680000019027,,Completions generated successfully. Ready for display in your IDE.,c9cc0aa8-92b9-4cad-8bf5-cec6ee86c9c1,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T13:59:23.165614,[1],"//This function takes an array l and returns an array l' such that // l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal // to the values of the corresponding indicies of l, but sorted. @@ -32117,7 +32117,7 @@ function sort_third(l){ } return x; }",52.64570000144886,,Completions generated successfully. Ready for display in your IDE.,95117b6b-61e4-4c5b-a703-eed822020a86,1,deepseek-ai/deepseek-coder-1.3b-base," -",30,1.0 +",30,1.0 2025-09-22T13:59:23.374012,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32154,7 +32154,7 @@ class Problem { return false; } }",208.55239999946207,,Completions generated successfully. Ready for display in your IDE.,557e2a25-31ce-4b54-a4a6-2d69683b26ae,1,deepseek-ai/deepseek-coder-1.3b-base," if (txt == null || txt.equals("""")) { -",171,0.9787 +",171,0.9787 2025-09-22T13:59:23.465892,[1],"//You are given a word. Your task is to find the closest vowel that stands between // two consonants from the right side of the word (case sensitive). // Vowels in the beginning and ending doesn't count. Return empty string if you didn't @@ -32192,7 +32192,7 @@ function get_closest_vowel(word){ return result[0];"," } }",91.6367000027094,,Completions generated successfully. Ready for display in your IDE.,2cfefcb5-2fe2-47c1-afde-9c812c176ed3,1,deepseek-ai/deepseek-coder-1.3b-base," } -",67,1.0 +",67,1.0 2025-09-22T13:59:23.662070,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -32264,7 +32264,7 @@ function minPath(grid, k){ return path; }",195.75239999539917,,Completions generated successfully. Ready for display in your IDE.,890e62d8-3100-4dfa-8448-3549f2ce416c,1,deepseek-ai/deepseek-coder-1.3b-base," //find smallest number in all elements of matrix -",166,0.8289 +",166,0.8289 2025-09-22T13:59:24.107732,[1],"from typing import Tuple def reverse_delete(s: str, c: str) -> Tuple[str, bool]: @@ -32289,7 +32289,7 @@ def reverse_delete(s: str, c: str) -> Tuple[str, bool]: if new_str == new_str_reverse: return (new_str,True) else: - return (new_str,False)",413,0.9929 + return (new_str,False)",413,0.9929 2025-09-22T13:59:24.212490,[1],"//Return the largest prime factor of n. Assume n > 1 and is not a prime. // >>> largest_prime_factor(13195) // 29 @@ -32306,7 +32306,7 @@ function largest_prime_factor(n){ } return result; }",104.44480000296608,,Completions generated successfully. Ready for display in your IDE.,27932dd9-6a1a-411e-be03-ddbcbe684c19,1,deepseek-ai/deepseek-coder-1.3b-base," var result = 2; -",83,1.0 +",83,1.0 2025-09-22T13:59:24.393943,[1],"//Write a function that accepts two arrays of strings and returns the array that has // total number of chars in the all strings of the array less than the other array. // if the two arrays have the same number of chars, return the first array. @@ -32337,7 +32337,7 @@ function total_match(lst1, lst2){ } ",181.4347999970778,,Completions generated successfully. Ready for display in your IDE.,8707f965-a996-4c83-9f5a-eaefc4f6e2b3,1,deepseek-ai/deepseek-coder-1.3b-base," sum_lst2 += item.length; -",156,1.0 +",156,1.0 2025-09-22T13:59:24.500148,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -32363,7 +32363,7 @@ function is_multiply_prime(a){ } }",106.32320000149775,,Completions generated successfully. Ready for display in your IDE.,cc7bc575-66ce-4ef1-b83f-983ed6fa9530,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",83,1.0 +",83,1.0 2025-09-22T13:59:24.596323,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32393,7 +32393,7 @@ class Problem { } }",95.36069999739993,,Completions generated successfully. Ready for display in your IDE.,23195242-d94e-4acf-829d-3d8f061ae9ea,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:24.689660,[1],"from typing import List, Optional def prod_signs(arr: List[int]) -> Optional[int]: @@ -32428,7 +32428,7 @@ def prod_signs(arr: List[int]) -> Optional[int]: sign_arr.append(-1)"," sign_arr.append(0) return sum(map(abs, arr)) * product(sign_arr)",93.05719999974826,,Completions generated successfully. Ready for display in your IDE.,c031527a-6a32-4688-b3a8-f61191ff35ba,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",70,1.0 +",70,1.0 2025-09-22T13:59:25.073203,[1],"//Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. // >>> fizz_buzz(50) // 0 @@ -32451,7 +32451,7 @@ function fizz_buzz(n){ ",383.5225000002538,,Completions generated successfully. Ready for display in your IDE.,a9233110-063c-4bc8-8f24-b4fcc1a39843,1,deepseek-ai/deepseek-coder-1.3b-base," // console.log(""i: "", i); if((i % 11 === 0 || i % 13 === 0)){ -",360,0.9577 +",360,0.9577 2025-09-22T13:59:25.342324,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32495,7 +32495,7 @@ class Problem { } }",269.2650999961188,,Completions generated successfully. Ready for display in your IDE.,88f738ae-5144-4ca5-83f0-307e18708961,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < sortedList.size(); i++) { -",245,1.0 +",245,1.0 2025-09-22T13:59:25.594737,[1],"from typing import Tuple def reverse_delete(s: str, c: str) -> Tuple[str, bool]: @@ -32519,7 +32519,7 @@ def reverse_delete(s: str, c: str) -> Tuple[str, bool]: ",252.22609999764245,,Completions generated successfully. Ready for display in your IDE.,88e0a0dc-9426-4b67-863a-48251082c6e9,1,deepseek-ai/deepseek-coder-1.3b-base," new_str_reverse = new_str[::-1] -",226,0.9964 +",226,0.9964 2025-09-22T13:59:25.754500,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -32577,7 +32577,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",159.65739999955986,,Completions generated successfully. Ready for display in your IDE.,c46779f8-6d6d-4447-a559-fc12a420c9a4,1,deepseek-ai/deepseek-coder-1.3b-base," elif x > 2.7: -",131,1.0 +",131,1.0 2025-09-22T13:59:25.968744,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32639,7 +32639,7 @@ class Problem { } }",213.941199996043,,Completions generated successfully. Ready for display in your IDE.,934e4ce0-b516-47b1-8cae-8c11e4d4a8f7,1,deepseek-ai/deepseek-coder-1.3b-base," if ((number % l) == 0) { -",184,1.0 +",184,1.0 2025-09-22T13:59:26.250245,[1],"//Given a string 'text', return its md5 hash equivalent string. // If 'text' is an empty string, return undefined. // >>> string_to_md5(""Hello world"") @@ -32652,7 +32652,7 @@ function string_to_md5(text){ }else { console.log (stringToMd5(text)); - ",258,0.8938 + ",258,0.8938 2025-09-22T13:59:26.438768,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -32719,7 +32719,7 @@ function numerical_letter_grade(grades){ return grades_array; }",188.66089999937685,,Completions generated successfully. Ready for display in your IDE.,b163ba02-0d91-4aaf-a040-9ae9886e9883,1,deepseek-ai/deepseek-coder-1.3b-base," grades_array.push(""B+""); -",160,1.0 +",160,1.0 2025-09-22T13:59:26.596776,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32751,7 +32751,7 @@ class Problem { } }",157.78950000094483,,Completions generated successfully. Ready for display in your IDE.,25f652d9-4285-4bbf-b4db-20e773fb194b,1,deepseek-ai/deepseek-coder-1.3b-base," b1 = -b1; -",131,1.0 +",131,1.0 2025-09-22T13:59:26.683533,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32774,7 +32774,7 @@ class Problem { public static boolean primeLength(String string) { return new java.math.BigInteger(String.valueOf(string.length())).isProbablePrime(100);",},86.36439999827417,,Completions generated successfully. Ready for display in your IDE.,2dfd405f-c3e7-4785-9bf4-70240716f7fd,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:26.839744,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -32829,7 +32829,7 @@ function by_length(arr){ return arr3; }",155.8835999967414,,Completions generated successfully. Ready for display in your IDE.,b19d5f22-8c39-45c5-b40a-f7df5cf88fc8,1,deepseek-ai/deepseek-coder-1.3b-base," arr3.push(""Two""); -",131,1.0 +",131,1.0 2025-09-22T13:59:27.083734,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32877,7 +32877,7 @@ class Problem { } }",243.24760000308743,,Completions generated successfully. Ready for display in your IDE.,4d0ea582-65ad-4abc-bf1a-663875d323fb,1,deepseek-ai/deepseek-coder-1.3b-base," int indexPlanet2 = planets.indexOf(planet2); -",212,1.0 +",212,1.0 2025-09-22T13:59:27.162490,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32917,7 +32917,7 @@ class Problem { return res; } }",78.99929999985034,,Completions generated successfully. Ready for display in your IDE.,f4de7919-a828-4856-922b-1849203649e6,1,deepseek-ai/deepseek-coder-1.3b-base," - ",51,1.0 + ",51,1.0 2025-09-22T13:59:27.255596,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -32948,7 +32948,7 @@ class Problem { } }",92.62149999995017,,Completions generated successfully. Ready for display in your IDE.,4c7374a5-c6fd-4a1d-9ca0-4e4645eba50a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:27.346609,[1],"//Input is a space-delimited string of numberals from 'zero' to 'nine'. // Valid choices are 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight' and 'nine'. // Return the string with numbers sorted from smallest to largest @@ -32972,7 +32972,7 @@ function sort_numbers(numbers){ numString = numString.join("" ""); return numString; }",91.45210000133375,,Completions generated successfully. Ready for display in your IDE.,96bf54db-0e62-499f-b1e4-175772549c15,1,deepseek-ai/deepseek-coder-1.3b-base,"}; -",49,0.9535 +",49,0.9535 2025-09-22T13:59:27.547825,[1],"//For a given array of input numbers, calculate Mean Absolute Deviation // around the mean of this dataset. // Mean Absolute Deviation is the average absolute difference between each @@ -32992,7 +32992,7 @@ function mean_absolute_deviation(numbers){ } ",200.384199997643,,Completions generated successfully. Ready for display in your IDE.,e460d226-c44e-41b3-850a-bdb0e04ab959,1,deepseek-ai/deepseek-coder-1.3b-base," sum+=Math.abs(numbers[i]); -",176,0.9433 +",176,0.9433 2025-09-22T13:59:27.709938,[1],"from typing import List def eat(number: int, need: int, remaining: int) -> List[int]: @@ -33029,7 +33029,7 @@ def eat(number: int, need: int, remaining: int) -> List[int]: Have fun :) """""" ",,162.61060000397265,,Completions generated successfully. Ready for display in your IDE.,973bbdf9-0250-4854-ab7d-b0b0abdcabae,1,deepseek-ai/deepseek-coder-1.3b-base," # TODO: Write your code here - pass",136,0.9293 + pass",136,0.9293 2025-09-22T13:59:27.946571,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33091,7 +33091,7 @@ class Problem { } }",236.19870000402443,,Completions generated successfully. Ready for display in your IDE.,cfe57f97-2f27-4c63-b147-07f3901b6506,1,deepseek-ai/deepseek-coder-1.3b-base," else if (item == 2 || item == 3) { -",205,0.987 +",205,0.987 2025-09-22T13:59:28.198118,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33131,7 +33131,7 @@ class Problem { } }",251.17599999794038,,Completions generated successfully. Ready for display in your IDE.,9bbea9fb-f557-46b1-8bf0-ea80ae2579cc,1,deepseek-ai/deepseek-coder-1.3b-base," Collections.sort(words, new Comparator() { -",223,1.0 +",223,1.0 2025-09-22T13:59:28.314902,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -33208,7 +33208,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",116.32470000040485,,Completions generated successfully. Ready for display in your IDE.,35600dc2-f6fa-42db-bb08-4a74a0fb0df4,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",88,1.0 +",88,1.0 2025-09-22T13:59:28.424921,[1],"def vowels_count(s: str) -> int: """"""Write a function vowels_count which takes a string representing a word as input and returns the number of vowels in the string. @@ -33231,7 +33231,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i return count ",109.5770999963861,,Completions generated successfully. Ready for display in your IDE.,f32cd09d-8ea9-43bb-bb45-d72303a486b6,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1 -",83,1.0 +",83,1.0 2025-09-22T13:59:28.602958,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33276,7 +33276,7 @@ class Problem { } }",178.38009999832138,,Completions generated successfully. Ready for display in your IDE.,6bec187f-ce51-40a6-bfe9-d447bb4009be,1,deepseek-ai/deepseek-coder-1.3b-base," for (int prime : primes) { -",152,1.0 +",152,1.0 2025-09-22T13:59:28.771776,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33326,7 +33326,7 @@ class Problem { return ret; } }",168.37710000254447,,Completions generated successfully. Ready for display in your IDE.,e0ee3f03-236e-4ad2-b2c9-b22b4aa1272b,1,deepseek-ai/deepseek-coder-1.3b-base," var ret = new ArrayList(); -",141,0.9387 +",141,0.9387 2025-09-22T13:59:28.922485,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -33384,7 +33384,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",150.23870000004536,,Completions generated successfully. Ready for display in your IDE.,0b10560e-cc1c-4afd-bc6e-8e89a02ccf6f,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('A') -",121,1.0 +",121,1.0 2025-09-22T13:59:28.989577,[1],"//triples_sum_to_zero takes an array of integers as an input. // it returns true if there are three distinct elements in the array that // sum to zero, and false otherwise. @@ -33411,7 +33411,7 @@ function triples_sum_to_zero(l){ } } return false;",,67.38290000066627,,Completions generated successfully. Ready for display in your IDE.,dd08ad3e-35c2-4ce1-b372-cb4e7948729e,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T13:59:29.146939,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -33466,7 +33466,7 @@ function by_length(arr){ return arr3; }",156.6878000012366,,Completions generated successfully. Ready for display in your IDE.,23dd7611-2e0f-4867-b261-bd5d1c159522,1,deepseek-ai/deepseek-coder-1.3b-base," arr3.push(""One""); -",131,1.0 +",131,1.0 2025-09-22T13:59:29.852821,[1],"//You are given a string s. // Your task is to check if the string is hapjs or not. // A string is hapjs if its length is at least 3 and every 3 consecutive letters are distinct @@ -33497,7 +33497,7 @@ function is_happy(s){ //check each letter in sequence with next two, if they're same then it isn't happy for(let i=0; i bool: """"""Your task is to write a function that returns true if a number x is a simple power of n and false in other cases. @@ -33551,7 +33551,7 @@ class Problem { elif x == 1: return True else: -",186,0.9583 +",186,0.9583 2025-09-22T13:59:30.600278,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -33597,7 +33597,7 @@ function histogram(test){ //finding highest value in dictionary to determine which keys are equal for (var key in count) { -",315,0.8125 +",315,0.8125 2025-09-22T13:59:30.750276,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -33655,7 +33655,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",149.74119999533286,,Completions generated successfully. Ready for display in your IDE.,5b3d84eb-91f7-4be9-8425-4804d83bc265,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('B') -",121,1.0 +",121,1.0 2025-09-22T13:59:30.855286,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -33681,7 +33681,7 @@ function is_multiply_prime(a){ }else{"," } }",104.33029999694554,,Completions generated successfully. Ready for display in your IDE.,3e3a739b-902c-4340-9af6-2aa8125777c3,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",82,1.0 +",82,1.0 2025-09-22T13:59:30.959283,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33730,7 +33730,7 @@ class Problem { } return valid_date;",},104.48460000043269,,Completions generated successfully. Ready for display in your IDE.,f4d8da4e-f306-452d-b3ae-fb9125a73109,1,deepseek-ai/deepseek-coder-1.3b-base," } -",77,1.0 +",77,1.0 2025-09-22T13:59:31.069284,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33773,7 +33773,7 @@ class Problem { } }",109.16569999972126,,Completions generated successfully. Ready for display in your IDE.,8b55fb2c-eb78-4424-84d9-064baf06596a,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",85,1.0 +",85,1.0 2025-09-22T13:59:31.772570,[1],"//You will be given a number in decimal form and your task is to convert it to // binary format. The function should return a string, with each character representing a binary // number. Each character in the string will be '0' or '1'. @@ -33789,7 +33789,7 @@ function decimal_to_binary(decimal){ ",703.0773000005865,,Completions generated successfully. Ready for display in your IDE.,d096d4e6-e07f-4018-a98c-b1560487791c,1,deepseek-ai/deepseek-coder-1.3b-base," let binary = """"; //create empty variable for binary while (decimal > 0 ){ //while loop that runs until decimal reaches zero if ((decimal % 2) === 0){ //if statement that checks remainder of decimal divided by two - binary += ""0""; ",676,0.9297 + binary += ""0""; ",676,0.9297 2025-09-22T13:59:31.842543,[1],"def prime_length(string: str) -> bool: """"""Write a function that takes a string and returns True if the string length is a prime number or False otherwise @@ -33813,7 +33813,7 @@ function decimal_to_binary(decimal){ else:"," pass ",70.07029999658698,,Completions generated successfully. Ready for display in your IDE.,633f43a5-f92a-49c5-acfb-665b9c664bf4,1,deepseek-ai/deepseek-coder-1.3b-base," - ",47,0.7477 + ",47,0.7477 2025-09-22T13:59:31.951548,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33857,7 +33857,7 @@ class Problem { }"," } }",106.98679999768501,,Completions generated successfully. Ready for display in your IDE.,9ef4da64-a815-4277-9c0a-c11e62abb792,1,deepseek-ai/deepseek-coder-1.3b-base," return """"; -",79,1.0 +",79,1.0 2025-09-22T13:59:32.025546,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -33895,7 +33895,7 @@ function even_odd_palindrome(n){ } } return [even, odd];",,73.88690000516362,,Completions generated successfully. Ready for display in your IDE.,d7d6d331-8c07-4339-9044-d37de990dbca,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:59:32.114543,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33915,7 +33915,7 @@ class Problem { } }",88.12229999603005,,Completions generated successfully. Ready for display in your IDE.,c5e70b33-cd74-47b1-8838-f6a660b6ece4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:59:32.220542,[1],"//Check if in given array of numbers, are any two numbers closer to each other than // given threshold. // >>> has_close_elements([1.0, 2.0, 3.0], 0.5) @@ -33932,7 +33932,7 @@ function has_close_elements(numbers, threshold){ } ",106.45049999584444,,Completions generated successfully. Ready for display in your IDE.,9d07738c-11b9-442d-a090-8cfb119d777c,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",83,1.0 +",83,1.0 2025-09-22T13:59:32.390547,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -33954,7 +33954,7 @@ class Problem { } }",169.4777999946382,,Completions generated successfully. Ready for display in your IDE.,b9ad06b1-24ee-42fb-81dd-a9e0c86460a7,1,deepseek-ai/deepseek-coder-1.3b-base," String s = Long.toString(i); -",145,1.0 +",145,1.0 2025-09-22T13:59:32.620156,[1],"from typing import List def count_nums(arr: List[int]) -> int: @@ -33984,7 +33984,7 @@ def count_nums(arr: List[int]) -> int: return n ",228.95080000307644,,Completions generated successfully. Ready for display in your IDE.,170878ca-7695-46d0-8fc8-4a0b56d0cd0f,1,deepseek-ai/deepseek-coder-1.3b-base," if s > 0: -",104,1.0 +",104,1.0 2025-09-22T13:59:32.696708,[1],"from typing import List, Tuple def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: @@ -34017,7 +34017,7 @@ def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: print(find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]))",76.87949999672128,,Completions generated successfully. Ready for display in your IDE.,dd97bc89-0660-4666-993c-33d749f2f4b6,1,deepseek-ai/deepseek-coder-1.3b-base,"2 -",49,1.0 +",49,1.0 2025-09-22T13:59:32.909839,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34066,7 +34066,7 @@ class Problem { } }",212.5746999954572,,Completions generated successfully. Ready for display in your IDE.,48e803e4-9618-401a-a930-9d1c30695ec6,1,deepseek-ai/deepseek-coder-1.3b-base," for (String key : hm.keySet()) { -",184,1.0 +",184,1.0 2025-09-22T13:59:33.017935,[1],"from typing import Dict def check_dict_case(dict: Dict[str, str]) -> bool: @@ -34094,7 +34094,7 @@ def check_dict_case(dict: Dict[str, str]) -> bool: if any(not x.isupper() for x in dict.keys()):"," return True ",108.23600000003353,,Completions generated successfully. Ready for display in your IDE.,b2eaba83-2f4f-49d4-81d2-19f258c83258,1,deepseek-ai/deepseek-coder-1.3b-base," return False -",74,1.0 +",74,1.0 2025-09-22T13:59:33.122405,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34142,7 +34142,7 @@ class Problem { } }",104.1830000031041,,Completions generated successfully. Ready for display in your IDE.,ef43db08-67b3-4c01-834a-85bc08e63944,1,deepseek-ai/deepseek-coder-1.3b-base," } -",77,1.0 +",77,1.0 2025-09-22T13:59:33.165948,[1],"def fib(n: int) -> int: """"""Return n-th Fibonacci number. >>> fib(10) @@ -34157,7 +34157,7 @@ class Problem { f0, f1 = f1, f0 + f1 return f1 -",42.75439999764785,,Completions generated successfully. Ready for display in your IDE.,b180014b-5014-4e94-9266-0ff14b97e49e,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",42.75439999764785,,Completions generated successfully. Ready for display in your IDE.,b180014b-5014-4e94-9266-0ff14b97e49e,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:59:33.231950,[1],"//Given array of numbers (of at least two elements), apply a linear transform to that array, // such that the smallest number will become 0 and the largest will become 1 // >>> rescale_to_unit([1.0, 2.0, 3.0, 4.0, 5.0]) @@ -34171,7 +34171,7 @@ function rescale_to_unit(numbers){ unit.push((numbers[i] - min) / sub); } return unit;",,65.55200000002515,,Completions generated successfully. Ready for display in your IDE.,6cbed3bc-7b52-4a67-a9a9-14eac6624a96,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:59:33.299947,[1],"//You are given an array of numbers. // You need to return the sum of squared numbers in the given array, // round each element in the array to the upper int(Ceiling) first. @@ -34188,7 +34188,7 @@ function rescale_to_unit(numbers){ // 6 function sum_squares(lst){ return lst.map(num => Math.ceil(num)).reduce((sum, num) => sum + num ** 2, 0);",,68.1966000047396,,Completions generated successfully. Ready for display in your IDE.,58ebb9bc-06ad-4086-8f5e-ecb801ca2483,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:59:33.478948,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34266,7 +34266,7 @@ class Problem { } }",178.08769999828655,,Completions generated successfully. Ready for display in your IDE.,efedbc4d-be71-48b2-a267-9312d58ef3fd,1,deepseek-ai/deepseek-coder-1.3b-base," num -= 1000; -",147,1.0 +",147,1.0 2025-09-22T13:59:33.805951,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34286,7 +34286,7 @@ class Problem { } }",327.19250000081956,,Completions generated successfully. Ready for display in your IDE.,d0aaf51f-0bc4-4a33-ad50-cca7a8f5d749,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < Math.min(a.length(),b.length()); ++i){ -",303,1.0 +",303,1.0 2025-09-22T13:59:33.883949,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -34313,7 +34313,7 @@ function next_smallest(lst){ return second_smallest; }",77.91079999879003,,Completions generated successfully. Ready for display in your IDE.,7249b2ee-b3d6-4792-85b9-45556b53181a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",52,1.0 +",52,1.0 2025-09-22T13:59:33.946952,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34340,7 +34340,7 @@ class Problem { return p; } }",61.973100004252046,,Completions generated successfully. Ready for display in your IDE.,4fcb13d6-21a6-4d27-8c29-f3879700756f,1,deepseek-ai/deepseek-coder-1.3b-base," -",33,1.0 +",33,1.0 2025-09-22T13:59:34.019951,[1],"//A simple program which should return the value of x if n is // a prime number and should return the value of y otherwise. // Examples: @@ -34358,7 +34358,7 @@ function x_or_y(n, x, y){ return x; }",73.12719999754336,,Completions generated successfully. Ready for display in your IDE.,2c56c849-9e02-496a-b908-0290bbbc9ccc,1,deepseek-ai/deepseek-coder-1.3b-base," } -",51,1.0 +",51,1.0 2025-09-22T13:59:34.154949,[1],"from typing import List def get_odd_collatz(n: int) -> List[int]: @@ -34392,7 +34392,7 @@ def get_odd_collatz(n: int) -> List[int]: ",135.11060000018915,,Completions generated successfully. Ready for display in your IDE.,85cf0d2c-cd28-4161-bbfb-89f458b1597c,1,deepseek-ai/deepseek-coder-1.3b-base," current_num = n -",111,1.0 +",111,1.0 2025-09-22T13:59:34.298951,[1],"from typing import Union, List def split_words(txt: str) -> Union[List[str], int]: @@ -34415,7 +34415,7 @@ def split_words(txt: str) -> Union[List[str], int]: return len([letter for letter in txt if letter in alphabet and alphabet.index(letter) % 2 == 1]) ",143.60669999587117,,Completions generated successfully. Ready for display in your IDE.,4ce22756-c2fd-4cb2-bf7f-4b8da219a9ba,1,deepseek-ai/deepseek-coder-1.3b-base," if ' ' in txt: -",114,0.9796 +",114,0.9796 2025-09-22T13:59:34.474198,[1],"def largest_divisor(n: int) -> int: """""" For a given number n, find the largest number that divides n evenly, smaller than n >>> largest_divisor(15) @@ -34425,7 +34425,7 @@ def split_words(txt: str) -> Union[List[str], int]: return divisor ",174.7255000009318,,Completions generated successfully. Ready for display in your IDE.,e67739b0-d775-403d-b890-9caa3454c7b1,1,deepseek-ai/deepseek-coder-1.3b-base," while not (n % divisor == 0): -",153,0.9946 +",153,0.9946 2025-09-22T13:59:34.566198,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34452,7 +34452,7 @@ class Problem { return Problem.fib((n - 1)) + Problem.fib((n - 2));"," } }",92.13820000150008,,Completions generated successfully. Ready for display in your IDE.,4df20da9-7c94-494d-ae85-f7ddaf04ee2d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:34.692632,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34486,7 +34486,7 @@ class Problem { } }",126.1305000007269,,Completions generated successfully. Ready for display in your IDE.,f12ec6d8-eada-4306-b0fe-c7a4e67c169a,1,deepseek-ai/deepseek-coder-1.3b-base," int count = 0; -",103,1.0 +",103,1.0 2025-09-22T13:59:34.843311,[1],"//Write a function count_nums which takes an array of integers and returns // the number of elements which has a sum of digits > 0. // If a number is negative, then its first signed digit will be negative: @@ -34520,7 +34520,7 @@ function count_nums(arr){ return count; }",149.55429999827174,,Completions generated successfully. Ready for display in your IDE.,c8537621-1f06-4b82-9685-6f4268829224,1,deepseek-ai/deepseek-coder-1.3b-base," if (Number(str)) { -",125,0.9647 +",125,0.9647 2025-09-22T13:59:34.930312,[1],"//I think we all remember that feeling when the result of some long-awaited // event is finally known. The feelings and thoughts you have at that moment are // definitely worth noting down and comparing. @@ -34543,7 +34543,7 @@ function compare(game, guess){ } }","} ",87.17099999921629,,Completions generated successfully. Ready for display in your IDE.,1629f8ce-b2c3-465c-83ad-2680be80a5e4,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",64,0.9863 +",64,0.9863 2025-09-22T13:59:34.995312,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34566,7 +34566,7 @@ class Problem { } return max; }",,64.26539999665692,,Completions generated successfully. Ready for display in your IDE.,0c7810de-dd1e-4e95-b8dc-b6d8697f50ee,1,deepseek-ai/deepseek-coder-1.3b-base," -}",40,1.0 +}",40,1.0 2025-09-22T13:59:35.305316,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34583,7 +34583,7 @@ class Problem { public static String concatenate(ArrayList strings) { "," } }",310.4987999977311,,Completions generated successfully. Ready for display in your IDE.,cf86393d-3498-4d46-99e0-af2a0f077bc1,1,deepseek-ai/deepseek-coder-1.3b-base," return strings.stream().reduce("""", (s1, s2) -> s1 + s2); -",288,0.9869 +",288,0.9869 2025-09-22T13:59:35.390324,[1],"//Given a positive integer N, return the total sum of its digits in binary. // Example // >>> solve(1000) @@ -34605,7 +34605,7 @@ function solve(N){ total += parseInt(digit);"," return total.toString(2); }",84.77750000020023,,Completions generated successfully. Ready for display in your IDE.,766927a3-a730-4a51-bfce-53c5b9fc9729,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:35.553310,[1],"def triangle_area(a: int, b: int, c: int) -> float: """""" Given the lengths of the three sides of a triangle. Return the area of @@ -34626,7 +34626,7 @@ function solve(N){ ",161.92700000101468,,Completions generated successfully. Ready for display in your IDE.,916a50e2-2415-41db-a2c1-e01ca9eb728b,1,deepseek-ai/deepseek-coder-1.3b-base," return round(area, 2) -",134,1.0 +",134,1.0 2025-09-22T13:59:35.637312,[1],"//The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fibfib(0) == 0 // fibfib(1) == 0 @@ -34652,7 +34652,7 @@ function fibfib(n){ ",84.17890000418993,,Completions generated successfully. Ready for display in your IDE.,406b5588-0071-46b3-a687-1b8a5ced94a7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",60,1.0 +",60,1.0 2025-09-22T13:59:35.712313,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34673,7 +34673,7 @@ class Problem { .collect(Collectors.toSet()) ); }",,74.36109999980545,,Completions generated successfully. Ready for display in your IDE.,bfa623c0-802d-4cf6-b3c5-9a8fc55735ed,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T13:59:36.381313,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -34723,7 +34723,7 @@ function int_to_mini_roman(number){ function mini_roman(){ //your code here... - ",644,0.966 + ",644,0.966 2025-09-22T13:59:36.580312,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34774,7 +34774,7 @@ class Problem { } }",199.5810000007623,,Completions generated successfully. Ready for display in your IDE.,99f5499f-d7fa-4156-af23-162b79aa5155,1,deepseek-ai/deepseek-coder-1.3b-base," if (tokens.length != 2) { -",172,1.0 +",172,1.0 2025-09-22T13:59:36.961181,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34803,7 +34803,7 @@ class Problem { }",379.5810000010533,,Completions generated successfully. Ready for display in your IDE.,873a88cb-3cdd-4bdd-b99a-e87660181672,1,deepseek-ai/deepseek-coder-1.3b-base," if ((arr.get(i).longValue()) >= arr.get(i-1).longValue()) continue; else { -",353,0.9519 +",353,0.9519 2025-09-22T13:59:37.099882,[1],"def car_race_collision(n: int) -> int: """""" Imagine a road that's a perfectly straight infinitely long line. @@ -34824,7 +34824,7 @@ class Problem { count += 1 return count",138.65619999705814,,Completions generated successfully. Ready for display in your IDE.,2fdbecb4-ed73-4057-9ddc-1559d8f6b617,1,deepseek-ai/deepseek-coder-1.3b-base," // 2 count = 0 -",114,1.0 +",114,1.0 2025-09-22T13:59:37.194542,[1],"def is_equal_to_sum_even(n: int) -> bool: """"""Evaluate whether the given number n can be written as the sum of exactly 4 positive even numbers Example @@ -34839,7 +34839,7 @@ class Problem { ",94.86179999657907,,Completions generated successfully. Ready for display in your IDE.,513e8522-b0d8-4763-bc69-bffcd634cf8d,1,deepseek-ai/deepseek-coder-1.3b-base," return False -",73,1.0 +",73,1.0 2025-09-22T13:59:37.395754,[1],"//Given array of numbers (of at least two elements), apply a linear transform to that array, // such that the smallest number will become 0 and the largest will become 1 // >>> rescale_to_unit([1.0, 2.0, 3.0, 4.0, 5.0]) @@ -34854,7 +34854,7 @@ function rescale_to_unit(numbers){ return unit; } ",200.66339999902993,,Completions generated successfully. Ready for display in your IDE.,8b4e0838-4c45-482e-b2cf-042487dc0d71,1,deepseek-ai/deepseek-coder-1.3b-base," var min = Math.min.apply(Math, numbers); -",178,1.0 +",178,1.0 2025-09-22T13:59:37.491661,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -34909,7 +34909,7 @@ function by_length(arr){ return arr3; }",95.92669999983627,,Completions generated successfully. Ready for display in your IDE.,3edc1426-28a2-41b0-83af-fc65b8b60d01,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:37.623605,[1],"//Given a positive integer n, return the product of the odd digits. // Return 0 if all digits are even. // For example: @@ -34942,7 +34942,7 @@ function digits(n){ ",131.79759999911766,,Completions generated successfully. Ready for display in your IDE.,cdeec168-0727-4cf8-adb6-81a2a8083742,1,deepseek-ai/deepseek-coder-1.3b-base," console.log(""Error""); -",111,0.9455 +",111,0.9455 2025-09-22T13:59:37.708336,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -34970,7 +34970,7 @@ class Problem { public static boolean iscube(long a) { return a==0l||(a>0l&&Math.pow(Math.round(Math.pow(a, 1.0/3.0)), 3)==a)||(a<0l&&Math.pow(Math.round(Math.pow(-a, 1.0/3.0)), 3)==-a);",},84.38870000099996,,Completions generated successfully. Ready for display in your IDE.,497a753c-37fc-4d72-8620-ea71c5191754,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:59:37.801773,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35021,7 +35021,7 @@ class Problem { } return ""Yes"";",},93.52649999345886,,Completions generated successfully. Ready for display in your IDE.,6e17ee61-4c14-4922-bde6-902260dfe74d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T13:59:37.971334,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35068,7 +35068,7 @@ class Problem { } }",168.75260000233538,,Completions generated successfully. Ready for display in your IDE.,b2872310-f711-49fd-8165-19f45f6f5aa3,1,deepseek-ai/deepseek-coder-1.3b-base," long level = 0l; -",136,1.0 +",136,1.0 2025-09-22T13:59:38.108806,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -35098,7 +35098,7 @@ function parse_nested_parens(paren_string){ ",137.47250000596978,,Completions generated successfully. Ready for display in your IDE.,fc9a4ffc-f601-4066-844a-b2850981d127,1,deepseek-ai/deepseek-coder-1.3b-base," maxlevel = level; -",113,1.0 +",113,1.0 2025-09-22T13:59:38.305962,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35163,7 +35163,7 @@ class Problem { } }",196.46510000166018,,Completions generated successfully. Ready for display in your IDE.,b276478a-80c3-49bb-97ec-639ab06a05c1,1,deepseek-ai/deepseek-coder-1.3b-base," letterGrades.add(""C+""); -",164,1.0 +",164,1.0 2025-09-22T13:59:38.380438,[1],"//Return 2^n modulo p (be aware of numerics). // >>> modp(3, 5) // 3 @@ -35185,7 +35185,7 @@ function modp(n, p){ ",74.85900000028778,,Completions generated successfully. Ready for display in your IDE.,e567d1b3-1bcc-48d4-b5f2-813051a990e1,1,deepseek-ai/deepseek-coder-1.3b-base," } -",53,1.0 +",53,1.0 2025-09-22T13:59:38.601242,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35216,7 +35216,7 @@ class Problem { } }",219.96219999709865,,Completions generated successfully. Ready for display in your IDE.,e5539ace-8fd6-4448-b378-1047b9e0497f,1,deepseek-ai/deepseek-coder-1.3b-base," int res_num = x_num * n_num; -",194,1.0 +",194,1.0 2025-09-22T13:59:38.728823,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35266,7 +35266,7 @@ class Problem { } }",127.30390000069747,,Completions generated successfully. Ready for display in your IDE.,9818c6ca-36a6-4549-ba6f-0ff648f8cc73,1,deepseek-ai/deepseek-coder-1.3b-base," case 'i': -",100,1.0 +",100,1.0 2025-09-22T13:59:38.786317,[1],"from typing import Dict def check_dict_case(dict: Dict[str, str]) -> bool: @@ -35293,7 +35293,7 @@ def check_dict_case(dict: Dict[str, str]) -> bool: return False return True ",56.84130000008736,,Completions generated successfully. Ready for display in your IDE.,8408e39e-0de0-49f3-af4b-a8da90f5f477,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,0.8678 +",32,0.8678 2025-09-22T13:59:38.875166,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -35317,7 +35317,7 @@ function vowels_count(s){ } ",88.98539999790955,,Completions generated successfully. Ready for display in your IDE.,3c89c1a0-c89d-4103-9f99-37a9ac69489a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",46,1.0 +",46,1.0 2025-09-22T13:59:39.122212,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35382,7 +35382,7 @@ class Problem { } }",247.43579999631038,,Completions generated successfully. Ready for display in your IDE.,3520d417-eca3-474f-963b-388836aeec1c,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (currentGrade > 1.3f) { -",216,1.0 +",216,1.0 2025-09-22T13:59:39.330666,[1],"from typing import List def Strongest_Extension(class_name: str, extensions: List[str]) -> str: @@ -35411,7 +35411,7 @@ def Strongest_Extension(class_name: str, extensions: List[str]) -> str: best_extension = extension return class_name + '.' + best_extension",207.66359999834094,,Completions generated successfully. Ready for display in your IDE.,3c506ad3-2425-4b6a-8267-c7725390f46b,1,deepseek-ai/deepseek-coder-1.3b-base," best_strength = 0 best_extension = '' -",181,0.9612 +",181,0.9612 2025-09-22T13:59:39.507006,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -35439,7 +35439,7 @@ function next_smallest(lst){ }",},175.83400000148686,,Completions generated successfully. Ready for display in your IDE.,3a88e09e-9464-4cd5-8c6b-22bd80d0380d,1,deepseek-ai/deepseek-coder-1.3b-base,"else{ return second_smallest; } -",149,1.0 +",149,1.0 2025-09-22T13:59:39.665395,[1],"//Given a positive integer n, return a sorted array that has the odd numbers in collatz sequence. // The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined // as follows: start with any positive integer n. Then each term is obtained from the @@ -35482,7 +35482,7 @@ function get_odd_collatz(n){ } ",158.3165000047302,,Completions generated successfully. Ready for display in your IDE.,11a7b5c3-b22e-4db0-9a4f-70dec137a61a,1,deepseek-ai/deepseek-coder-1.3b-base," var odd_numbers = [] -",131,0.9292 +",131,0.9292 2025-09-22T13:59:39.934029,[1],"//Given a string s, count the number of uppercase vowels in even indices. // For example: // >>> count_upper(""aBCdEf"") @@ -35501,7 +35501,7 @@ function count_upper(s){ return result; }",268.459899998561,,Completions generated successfully. Ready for display in your IDE.,65f6fe7a-9f12-4f98-a84c-d76415e58c83,1,deepseek-ai/deepseek-coder-1.3b-base," if ((vowel).indexOf(s[index]) !== -1) { -",238,0.9781 +",238,0.9781 2025-09-22T13:59:40.056990,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35546,7 +35546,7 @@ class Problem { } }",122.94340000516968,,Completions generated successfully. Ready for display in your IDE.,47d04767-50e1-497c-8ae9-71679bbf7d05,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",92,1.0 +",92,1.0 2025-09-22T13:59:40.153738,[1],"//In this problem, you will implement a function that takes two arrays of numbers, // and determines whether it is possible to perform an exchange of elements // between them to make lst1 an array of only even numbers. @@ -35585,7 +35585,7 @@ function exchange(lst1, lst2){ return ""NO"";","} ",95.6196999977692,,Completions generated successfully. Ready for display in your IDE.,f07abe3f-a62d-443e-b926-ab59340c7333,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:40.262742,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35612,7 +35612,7 @@ class Problem { } }",108.63680000329623,,Completions generated successfully. Ready for display in your IDE.,4e6f90a6-ccaf-43a0-bdc7-7bfa69dd8919,1,deepseek-ai/deepseek-coder-1.3b-base," } else - ",83,0.9867 + ",83,0.9867 2025-09-22T13:59:40.369629,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -35637,7 +35637,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: groups.append(group)"," return groups ",107.15379999601282,,Completions generated successfully. Ready for display in your IDE.,34f68f71-bc92-400d-ac6c-fd52e777f21e,1,deepseek-ai/deepseek-coder-1.3b-base," group = '' -",84,1.0 +",84,1.0 2025-09-22T13:59:40.484522,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -35686,7 +35686,7 @@ function check_dict_case(dict){ return true; }",114.28909999813186,,Completions generated successfully. Ready for display in your IDE.,f0f152e5-2b37-482e-a605-ddb873f8f2c7,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",88,1.0 +",88,1.0 2025-09-22T13:59:40.671431,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35728,7 +35728,7 @@ class Problem { } }",187.06720000045607,,Completions generated successfully. Ready for display in your IDE.,52faabc8-451d-42c4-9992-48c27e2b7997,1,deepseek-ai/deepseek-coder-1.3b-base," String s3 = s1+s2; -",161,1.0 +",161,1.0 2025-09-22T13:59:40.741447,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -35777,7 +35777,7 @@ function prime_fib(n){ } return fib(fibNum - 1); }",69.77750000078231,,Completions generated successfully. Ready for display in your IDE.,2b6e7433-c657-4a5a-8514-3ffd518bf7a1,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T13:59:41.459703,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -35860,7 +35860,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i # return b # else: # return None - ",688,0.9495 + ",688,0.9495 2025-09-22T13:59:41.583162,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -35900,7 +35900,7 @@ function match_parens(lst){ return 'No'; }",122.98310000187485,,Completions generated successfully. Ready for display in your IDE.,ba6b7ba0-b536-4a7d-8f0f-361b8c70bf84,1,deepseek-ai/deepseek-coder-1.3b-base," return false; - ",99,0.9889 + ",99,0.9889 2025-09-22T13:59:41.997639,[1],"//Given a string of words, return an array of words split on whitespace, if no whitespaces exists in the text you // should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the // alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 @@ -35930,7 +35930,7 @@ function split_words(txt){ }",414.26639999554027,,Completions generated successfully. Ready for display in your IDE.,cf178479-ba16-4f15-918c-4362c1a849cb,1,deepseek-ai/deepseek-coder-1.3b-base," const letter = txt[i]; if (letter === letter.toLowerCase() && /^[A-Z]/.test(letter)) { -",390,0.9372 +",390,0.9372 2025-09-22T13:59:42.210613,[1],"def is_multiply_prime(a: int) -> bool: """"""Write a function that returns true if the given number is the multiplication of 3 prime numbers and false otherwise. @@ -35957,7 +35957,7 @@ function split_words(txt){ ",211.9862000035937,,Completions generated successfully. Ready for display in your IDE.,7cab6ac0-58a6-49e1-8d41-670cc6958943,1,deepseek-ai/deepseek-coder-1.3b-base," if len(prime_factors) == 3: -",189,1.0 +",189,1.0 2025-09-22T13:59:42.328831,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -35986,7 +35986,7 @@ class Problem { } }",118.40150000352878,,Completions generated successfully. Ready for display in your IDE.,f7001554-b9d5-4a62-acb4-37b481828d2f,1,deepseek-ai/deepseek-coder-1.3b-base," return sqrt; -",93,1.0 +",93,1.0 2025-09-22T13:59:42.848793,[1],"//Given array of numbers (of at least two elements), apply a linear transform to that array, // such that the smallest number will become 0 and the largest will become 1 // >>> rescale_to_unit([1.0, 2.0, 3.0, 4.0, 5.0]) @@ -36003,7 +36003,7 @@ function rescale_to_unit(numbers){ if (!sub || !Number.isFinite(sub)) throw new Error('Invalid input'); //check whether there is any NaN or Infinity in this list var unit = []; -",492,0.907 +",492,0.907 2025-09-22T13:59:42.915221,[1],"//You're given an array of deposit and withdrawal operations on a bank account that starts with // zero balance. Your task is to detect if at any point the balance of account fallls below zero, and // at that point function should return true. Otherwise it should return false. @@ -36020,7 +36020,7 @@ function below_zero(operations){ } } return false;",,66.59870000294177,,Completions generated successfully. Ready for display in your IDE.,733ae7ab-dc80-4f21-8a86-2119b8d2a34a,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:59:43.052208,[1],"from typing import List, Optional def prod_signs(arr: List[int]) -> Optional[int]: @@ -36055,7 +36055,7 @@ def prod_signs(arr: List[int]) -> Optional[int]: sign_arr.append(0) return sum(map(abs, arr)) * product(sign_arr)",136.52830000501126,,Completions generated successfully. Ready for display in your IDE.,09b8eae0-513a-469b-9a82-338b59becaa2,1,deepseek-ai/deepseek-coder-1.3b-base," elif num < 0: -",104,1.0 +",104,1.0 2025-09-22T13:59:43.378987,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36089,7 +36089,7 @@ class Problem { if (current_depth > max_depth) { max_depth = current_depth; } -",301,1.0 +",301,1.0 2025-09-22T13:59:43.457238,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36119,7 +36119,7 @@ class Problem { return b.toString(); }",,77.79840000148397,,Completions generated successfully. Ready for display in your IDE.,54da171c-7f74-4cf7-830b-bf045311231b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",53,0.964 +",53,0.964 2025-09-22T13:59:44.152873,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36146,7 +36146,7 @@ class Problem { private static final Scanner scanner = new Scanner(System.in); public static void main(String[] args) throws IOException { - BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(System.",668,0.9934 + BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(System.",668,0.9934 2025-09-22T13:59:44.475261,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36195,7 +36195,7 @@ class Problem { } }",322.6725999993505,,Completions generated successfully. Ready for display in your IDE.,4065261e-7b09-4e9d-aadb-c49ae6e2d46a,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (month == 4 || month == 6 || month == 9 || month == 11) { -",290,1.0 +",290,1.0 2025-09-22T13:59:44.583906,[1],"//You are given a string representing a sentence, // the sentence contains some words separated by a space, // and you have to return a string that contains the words from the original sentence, @@ -36225,7 +36225,7 @@ function words_in_sentence(sentence){ }).join(' '); }",107.72510000242619,,Completions generated successfully. Ready for display in your IDE.,73d13699-cf19-46e6-95b8-c3b7ab157d78,1,deepseek-ai/deepseek-coder-1.3b-base," return null; -",82,0.9332 +",82,0.9332 2025-09-22T13:59:44.702797,[1],"//You are given a string s. // Your task is to check if the string is hapjs or not. // A string is hapjs if its length is at least 3 and every 3 consecutive letters are distinct @@ -36253,7 +36253,7 @@ function is_happy(s){ } return happy; }",118.96669999987353,,Completions generated successfully. Ready for display in your IDE.,068669be-fc1c-40bf-8ed7-c6e81fca647f,1,deepseek-ai/deepseek-coder-1.3b-base," let happy = true; -",93,0.9548 +",93,0.9548 2025-09-22T13:59:44.794849,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -36282,7 +36282,7 @@ function f(n){ return result; }",91.81580000586109,,Completions generated successfully. Ready for display in your IDE.,be43e98d-ac4f-47f1-8383-1511240bf85b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",67,1.0 +",67,1.0 2025-09-22T13:59:44.963830,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -36354,7 +36354,7 @@ function minPath(grid, k){ return path; }",168.30789999949047,,Completions generated successfully. Ready for display in your IDE.,7e0d3bb5-5302-41d1-aed7-7e640d9b6775,1,deepseek-ai/deepseek-coder-1.3b-base," nextCell = curr; -",135,1.0 +",135,1.0 2025-09-22T13:59:45.278087,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -36426,7 +36426,7 @@ function minPath(grid, k){ return path; }",314.62740000279155,,Completions generated successfully. Ready for display in your IDE.,589b345e-5566-4685-8ecd-c514ecdd1b4c,1,deepseek-ai/deepseek-coder-1.3b-base," while(k-- !== 0 && currCell < 10 ** 9){ -",271,0.9379 +",271,0.9379 2025-09-22T13:59:45.389930,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -36468,7 +36468,7 @@ function compare_one(a, b){ } ",111.224599997513,,Completions generated successfully. Ready for display in your IDE.,4261dd7b-6f6f-4d18-9b3d-aaaa0d1bf3c8,1,deepseek-ai/deepseek-coder-1.3b-base," return b; -",84,1.0 +",84,1.0 2025-09-22T13:59:45.445644,[1],"def encrypt(s: str) -> str: """"""Create a function encrypt that takes a string as an argument and returns a string encrypted with the alphabet being rotated. @@ -36488,7 +36488,7 @@ function compare_one(a, b){ output_string += input_string[(input_string.index(i)+4)%26] return output_string ",54.936199994699564,,Completions generated successfully. Ready for display in your IDE.,1cb14596-07e2-4118-b973-d2ee5c54a004,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:59:45.584369,[1],"//Change numerical base of input number x to base. // return string representation after the conversion. // base numbers are less than 10. @@ -36504,7 +36504,7 @@ function change_base(x, base){ return change_base(q, base) + r.toString(); } ",138.3413000003202,,Completions generated successfully. Ready for display in your IDE.,d390fe9f-1b3a-4474-8169-1d1e0769fa0b,1,deepseek-ai/deepseek-coder-1.3b-base," const r = x % base; -",113,1.0 +",113,1.0 2025-09-22T13:59:45.724831,[1],"from typing import List def count_nums(arr: List[int]) -> int: @@ -36534,7 +36534,7 @@ def count_nums(arr: List[int]) -> int: return n ",140.17320000129985,,Completions generated successfully. Ready for display in your IDE.,0964da28-b3c2-4d6b-b1b5-ec8d0f4d2f84,1,deepseek-ai/deepseek-coder-1.3b-base," s += int(j) -",113,1.0 +",113,1.0 2025-09-22T13:59:45.802831,[1],"//Out of array of strings, return the longest one. Return the first one in case of multiple // strings of the same length. Return undefined in case the input array is empty. // >>> longest([]) @@ -36554,7 +36554,7 @@ function longest(strings){ } } return result;",,77.51000000280328,,Completions generated successfully. Ready for display in your IDE.,5b04de28-b8b3-4686-97c5-98ef13cbf2a6,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T13:59:45.899898,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36580,7 +36580,7 @@ class Problem { } }",96.59339999780059,,Completions generated successfully. Ready for display in your IDE.,67974fcb-4f60-4100-9d63-7efc943a2c3a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",65,1.0 +",65,1.0 2025-09-22T13:59:45.976963,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36605,7 +36605,7 @@ class Problem { } return ret; }",,77.15699999971548,,Completions generated successfully. Ready for display in your IDE.,3505bad2-6cd4-4ef6-af0b-0204d20f82a1,1,deepseek-ai/deepseek-coder-1.3b-base," -}",45,1.0 +}",45,1.0 2025-09-22T13:59:46.036065,[1],"//Implement a function that takes an non-negative integer and returns an array of the first n // integers that are prime numbers and less than n. // for example: @@ -36635,7 +36635,7 @@ function count_up_to(n){ return output; } ",58.19360000168672,,Completions generated successfully. Ready for display in your IDE.,40683558-4fd4-484c-8230-f6a9edb99eb0,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T13:59:46.158432,[1],"//You are given an array arr of integers and you need to return // sum of magnitudes of integers multiplied by product of all signs // of each number in the array, represented by 1, -1 or 0. @@ -36668,7 +36668,7 @@ function prod_signs(arr){ } ",122.43989999842597,,Completions generated successfully. Ready for display in your IDE.,40b6d7ba-e52b-4b36-8c19-9c17f788fd13,1,deepseek-ai/deepseek-coder-1.3b-base," var pro = 1; -",98,1.0 +",98,1.0 2025-09-22T13:59:46.215252,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -36693,7 +36693,7 @@ function string_xor(a, b){ result = arrayXOR.join(''); return result; }",56.83439999847906,,Completions generated successfully. Ready for display in your IDE.,86c647cf-d028-4e18-9247-3162c99a5f1b,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,0.5341 +",32,0.5341 2025-09-22T13:59:46.273145,[1],"//Given an array arr of integers and a positive integer k, return a sorted array // of length k with the maximum k numbers in arr. // Example 1: @@ -36713,7 +36713,7 @@ function maximum(arr, k){ var len = arr.length; if(len === 0) return null;"," return arr.sort((a, b) => a - b).slice(len - k); }",57.505699995090254,,Completions generated successfully. Ready for display in your IDE.,71ab17f2-c7d4-4699-9d1a-aa8f620b879d,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T13:59:46.391925,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36736,7 +36736,7 @@ class Problem { return max; } }",118.41329999879235,,Completions generated successfully. Ready for display in your IDE.,0bcdd95b-5762-4773-ad54-a1a6da2f290b,1,deepseek-ai/deepseek-coder-1.3b-base," Long max = null; -",93,0.9905 +",93,0.9905 2025-09-22T13:59:46.457865,[1],"//Return true if all numbers in the array l are below threshold t. // >>> below_threshold([1, 2, 4, 10], 100) // true @@ -36744,7 +36744,7 @@ class Problem { // false function below_threshold(l, t){ return l.every(x => x < t);",,66.02120000025025,,Completions generated successfully. Ready for display in your IDE.,843198e8-8a50-43da-900e-ba45b73322be,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:59:47.176664,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36792,7 +36792,7 @@ class Problem { copy.set(i, copy.get(j)); copy.set(j, temp); break; - ",688,0.9528 + ",688,0.9528 2025-09-22T13:59:47.342690,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36827,7 +36827,7 @@ class Problem { return ans; } }",165.53730000305222,,Completions generated successfully. Ready for display in your IDE.,8a48ee89-1ea5-4ad1-af45-f44ab55f461c,1,deepseek-ai/deepseek-coder-1.3b-base," int sz = lst.size(); -",140,1.0 +",140,1.0 2025-09-22T13:59:47.429916,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36850,7 +36850,7 @@ class Problem { } }",86.95230000012089,,Completions generated successfully. Ready for display in your IDE.,18dc5061-9b67-47c3-89b6-ce6539e7a738,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T13:59:47.600323,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36878,7 +36878,7 @@ class Problem { } }",170.65049999655457,,Completions generated successfully. Ready for display in your IDE.,317613bb-7357-4db3-9452-8909258d4252,1,deepseek-ai/deepseek-coder-1.3b-base," index = string.indexOf(substring); -",145,1.0 +",145,1.0 2025-09-22T13:59:47.823869,[1],"from typing import Optional def string_to_md5(text: str) -> Optional[str]: @@ -36893,7 +36893,7 @@ def string_to_md5(text: str) -> Optional[str]: if not text: return None else:",,223.33269999944605,,Completions generated successfully. Ready for display in your IDE.,01024c2e-0b19-49b8-ad7c-52653d059afb,1,deepseek-ai/deepseek-coder-1.3b-base," - return hashlib.md5(text.encode()).hexdigest()",200,1.0 + return hashlib.md5(text.encode()).hexdigest()",200,1.0 2025-09-22T13:59:48.123300,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36938,7 +36938,7 @@ class Problem { if ((Collections.min(copy)).equals(last)){ return true; } -",268,0.9212 +",268,0.9212 2025-09-22T13:59:48.253718,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -36988,7 +36988,7 @@ class Problem { } }",129.9384000012651,,Completions generated successfully. Ready for display in your IDE.,311364f0-36a0-4c96-a4d7-b3c6ecc98fd2,1,deepseek-ai/deepseek-coder-1.3b-base," case 'u': -",100,1.0 +",100,1.0 2025-09-22T13:59:48.449812,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37020,7 +37020,7 @@ class Problem { } }",196.00319999881322,,Completions generated successfully. Ready for display in your IDE.,51ec2233-c435-47e7-b535-360d51e071e8,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (c == ')') { -",156,1.0 +",156,1.0 2025-09-22T13:59:48.560909,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37085,7 +37085,7 @@ class Problem { } return letterGrades;",},110.66469999786932,,Completions generated successfully. Ready for display in your IDE.,8163f82c-d300-4c3d-a332-2ce2b460676d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",81,1.0 +",81,1.0 2025-09-22T13:59:48.667131,[1],"//Return array of prime factors of given integer in the order from smallest to largest. // Each of the factors should be arrayed number of times corresponding to how many times it appeares in factorization. // Input number should be equal to the product of all factors @@ -37108,7 +37108,7 @@ function factorize(n){ } ",106.30160000437172,,Completions generated successfully. Ready for display in your IDE.,a4359bfc-7de6-435d-893f-12d7137d4050,1,deepseek-ai/deepseek-coder-1.3b-base," } else{ -",83,1.0 +",83,1.0 2025-09-22T13:59:48.793142,[1],"//This function takes two positive numbers x and y and returns the // biggest even integer number that is in the range [x, y] inclusive. If // there's no such number, then the function should return -1. @@ -37127,7 +37127,7 @@ function choose_num(x, y){ ",125.78559999383288,,Completions generated successfully. Ready for display in your IDE.,50b474a6-2709-4591-9a97-3381478fc8de,1,deepseek-ai/deepseek-coder-1.3b-base," break; } -",101,0.9896 +",101,0.9896 2025-09-22T13:59:48.990341,[1],"//Given an array arr of integers and a positive integer k, return a sorted array // of length k with the maximum k numbers in arr. // Example 1: @@ -37148,7 +37148,7 @@ function maximum(arr, k){ if(len === 1) return arr; return arr.sort((a, b) => a - b).slice(len - k); }",197.2276000014972,,Completions generated successfully. Ready for display in your IDE.,e7f3fa0e-aef4-4cbf-bdbf-8c4a2b65f9eb,1,deepseek-ai/deepseek-coder-1.3b-base," let len = Math.min(k, arr.length); -",174,0.8343 +",174,0.8343 2025-09-22T13:59:49.086734,[1],"def is_multiply_prime(a: int) -> bool: """"""Write a function that returns true if the given number is the multiplication of 3 prime numbers and false otherwise. @@ -37174,7 +37174,7 @@ function maximum(arr, k){ ",95.8466000010958,,Completions generated successfully. Ready for display in your IDE.,1944de3b-7023-4a1b-8ffa-9bd7143b832b,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",72,1.0 +",72,1.0 2025-09-22T13:59:49.183341,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -37206,7 +37206,7 @@ function correct_bracketing(brackets){ } }",95.96289999899454,,Completions generated successfully. Ready for display in your IDE.,b194f529-ec8f-4ff6-a308-aac068297cbc,1,deepseek-ai/deepseek-coder-1.3b-base," break; -",73,0.9579 +",73,0.9579 2025-09-22T13:59:49.367582,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37243,7 +37243,7 @@ class Problem { } }",184.21349999698577,,Completions generated successfully. Ready for display in your IDE.,cb0abc55-d116-438a-a2b4-dea678f6228c,1,deepseek-ai/deepseek-coder-1.3b-base," if (open_paren > 0) { -",162,1.0 +",162,1.0 2025-09-22T13:59:50.069389,[1],"//You are given a word. Your task is to find the closest vowel that stands between // two consonants from the right side of the word (case sensitive). // Vowels in the beginning and ending doesn't count. Return empty string if you didn't @@ -37287,7 +37287,7 @@ function get_closest_vowel(word){ break; } } - console.log('here', v",676,0.9233 + console.log('here', v",676,0.9233 2025-09-22T13:59:50.332054,[1],"//You are given a positive integer n. You have to create an integer array a of length n. // For each i (1 ≤ i ≤ n), the value of a[i] = i * i - i + 1. // Return the number of triples (a[i], a[j], a[k]) of a where i < j < k, @@ -37315,7 +37315,7 @@ function get_max_triples(n){ } ",262.86500000423985,,Completions generated successfully. Ready for display in your IDE.,de9614df-04f9-4dea-a7c5-a1a68e21ac26,1,deepseek-ai/deepseek-coder-1.3b-base," for(let j = i+1; j k.matches(""^[A-Z]+$"")); return allLowerCase || allUpperCase; }",,81.97980000113603,,Completions generated successfully. Ready for display in your IDE.,e1eba5b0-9ab3-48b1-86cc-556402bf3c57,1,deepseek-ai/deepseek-coder-1.3b-base," -}",47,1.0 +}",47,1.0 2025-09-22T13:59:50.850172,[1],"//You have been tasked to write a function that receives // a hexadecimal number as a string and counts the number of hexadecimal // digits that are primes (prime number, or a prime, is a natural number @@ -37381,7 +37381,7 @@ function hex_key(num){ continue; }else { // console.log('test'); -",400,0.9185 +",400,0.9185 2025-09-22T13:59:50.973705,[1],"//For a given number n, find the largest number that divides n evenly, smaller than n // >>> largest_divisor(15) // 5 @@ -37395,7 +37395,7 @@ function largest_divisor(n){ return ld; }",123.60120000084862,,Completions generated successfully. Ready for display in your IDE.,9559f20f-5b5f-4bdc-ae23-51d8e464516c,1,deepseek-ai/deepseek-coder-1.3b-base," ld = i; -",101,1.0 +",101,1.0 2025-09-22T13:59:51.018146,[1],"from typing import List def total_match(lst1: List[str], lst2: List[str]) -> List[str]: @@ -37424,7 +37424,7 @@ def total_match(lst1: List[str], lst2: List[str]) -> List[str]: return lst1 return lst2 -",44.367399998009205,,Completions generated successfully. Ready for display in your IDE.,55e454d8-a76b-490f-8efb-423941fe8f37,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",44.367399998009205,,Completions generated successfully. Ready for display in your IDE.,55e454d8-a76b-490f-8efb-423941fe8f37,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:59:51.113444,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37457,7 +37457,7 @@ class Problem { } }",94.74629999749595,,Completions generated successfully. Ready for display in your IDE.,5d86b795-4f2c-4366-8959-8c9bbc5ca6d9,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:59:51.157263,[1],"from typing import List def below_zero(operations: List[int]) -> bool: @@ -37476,7 +37476,7 @@ def below_zero(operations: List[int]) -> bool: return False print(below_zero([1, 2, 3])) -print(below_zero([1, 2, -4, 5]))",43.77999999996973,,Completions generated successfully. Ready for display in your IDE.,8c9f668a-79df-468a-aa33-c249de663452,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +print(below_zero([1, 2, -4, 5]))",43.77999999996973,,Completions generated successfully. Ready for display in your IDE.,8c9f668a-79df-468a-aa33-c249de663452,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T13:59:51.403812,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37541,7 +37541,7 @@ class Problem { } }",246.2322000064887,,Completions generated successfully. Ready for display in your IDE.,9c6f2d4e-c671-48d7-905d-c6e8c698db72,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (currentGrade > 0.0f) { -",216,1.0 +",216,1.0 2025-09-22T13:59:51.794656,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37595,7 +37595,7 @@ class Problem { }",390.907300003164,,Completions generated successfully. Ready for display in your IDE.,8de4ab13-3730-4a3b-a654-0e28e8078414,1,deepseek-ai/deepseek-coder-1.3b-base," else if (arr.get(i) == minValue && i < minIndex) { minIndex = i; } - ",362,1.0 + ",362,1.0 2025-09-22T13:59:51.911035,[1],"//You are given an array of integers. // You need to find the largest prime value and return the sum of its digits. // Examples: @@ -37626,7 +37626,7 @@ function skjkasdkd(lst){ return sum; }",116.15219999657711,,Completions generated successfully. Ready for display in your IDE.,635f9ca2-f084-413a-a4b8-6407d66d1335,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",90,1.0 +",90,1.0 2025-09-22T13:59:52.063396,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -37694,7 +37694,7 @@ class Problem { } }",151.5988000028301,,Completions generated successfully. Ready for display in your IDE.,44d1af72-b302-4648-9c24-cb445db5bfdc,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",105,1.0 +",105,1.0 2025-09-22T13:59:52.216552,[1],"//Given an array of numbers, return whether or not they are sorted // in ascending order. If array has more than 1 duplicate of the same // number, return false. Assume no negative numbers and only integers. @@ -37736,7 +37736,7 @@ function is_sorted(lst){ } ",153.79620000021532,,Completions generated successfully. Ready for display in your IDE.,f2011498-1418-49ab-80b1-0dbccdfeeb45,1,deepseek-ai/deepseek-coder-1.3b-base," dupe_found = true; -",130,1.0 +",130,1.0 2025-09-22T13:59:52.483031,[1],"//A simple program which should return the value of x if n is // a prime number and should return the value of y otherwise. // Examples: @@ -37754,7 +37754,7 @@ function x_or_y(n, x, y){ return x; }",265.9473000021535,,Completions generated successfully. Ready for display in your IDE.,99d03aad-f403-4b2b-ab10-44e190c38c69,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=2; i < Math.sqrt(n); ++i ){ -",242,0.9653 +",242,0.9653 2025-09-22T13:59:52.735251,[1],"def encode(message: str) -> str: """""" Write a function that takes a message, and encodes in such a @@ -37780,7 +37780,7 @@ function x_or_y(n, x, y){ return ch.lower() return chr(ord(ch) + 2) return ch",,251.70410000282573,,Completions generated successfully. Ready for display in your IDE.,690c6e92-29f1-41ec-ae67-c4c8216f4ac2,1,deepseek-ai/deepseek-coder-1.3b-base," - return """".join([encode_vowel(ch) for ch in message])",226,0.9325 + return """".join([encode_vowel(ch) for ch in message])",226,0.9325 2025-09-22T13:59:52.985246,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -37835,7 +37835,7 @@ function by_length(arr){ return arr3; }",250.23979999969015,,Completions generated successfully. Ready for display in your IDE.,088b3efd-12da-4339-a82e-bf201045b1a0,1,deepseek-ai/deepseek-coder-1.3b-base," for(var i = 0; i < arr.length; i++){ -",223,1.0 +",223,1.0 2025-09-22T13:59:53.170641,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -37879,7 +37879,7 @@ function histogram(test){ } ",185.09690000064438,,Completions generated successfully. Ready for display in your IDE.,ea8583d1-32ee-43bb-b6f5-a5b7a26a97d3,1,deepseek-ai/deepseek-coder-1.3b-base," count[lst[i]] = 1; -",160,1.0 +",160,1.0 2025-09-22T13:59:53.455212,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -37919,7 +37919,7 @@ function find_max(words){ }",283.8666000025114,,Completions generated successfully. Ready for display in your IDE.,f96fbb30-0872-4ee4-ad91-9c8c74bf6f56,1,deepseek-ai/deepseek-coder-1.3b-base," for(var i=0; i= 50) { -",143,1.0 +",143,1.0 2025-09-22T13:59:53.801198,[1],"from typing import Dict def histogram(test: str) -> Dict[str, int]: @@ -38029,7 +38029,7 @@ def histogram(test: str) -> Dict[str, int]: if letters[letter] == most_count:"," return result ",174.23430000053486,,Completions generated successfully. Ready for display in your IDE.,bde1ade6-c1e8-47e0-b303-95a25bb53848,1,deepseek-ai/deepseek-coder-1.3b-base," result[letter] = letters[letter] -",149,1.0 +",149,1.0 2025-09-22T13:59:53.901250,[1],"from typing import List def triples_sum_to_zero(l: List[int]) -> bool: @@ -38056,7 +38056,7 @@ def triples_sum_to_zero(l: List[int]) -> bool: for k in range(j + 1, len(l)): if l[i] + l[j] + l[k] == 0: return True",,99.46759999729693,,Completions generated successfully. Ready for display in your IDE.,db34626e-d70d-4511-89b7-1142ec7fc5fc,1,deepseek-ai/deepseek-coder-1.3b-base," - return False",72,1.0 + return False",72,1.0 2025-09-22T13:59:54.035969,[1],"//Given a string of words, return an array of words split on whitespace, if no whitespaces exists in the text you // should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the // alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 @@ -38085,7 +38085,7 @@ function split_words(txt){ } }",135.23149999673478,,Completions generated successfully. Ready for display in your IDE.,c5a2efa6-482c-4d9f-8e21-b90665e2a54b,1,deepseek-ai/deepseek-coder-1.3b-base," let count = 0; -",102,0.9725 +",102,0.9725 2025-09-22T13:59:54.129979,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38122,7 +38122,7 @@ class Problem { } }",93.6437999989721,,Completions generated successfully. Ready for display in your IDE.,75074240-da91-4276-b731-8e017a68e59c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T13:59:54.242653,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -38194,7 +38194,7 @@ function minPath(grid, k){ return path; }",112.68980000022566,,Completions generated successfully. Ready for display in your IDE.,0b18728f-b0a7-434e-b793-36d52e5c3bfd,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T13:59:54.372675,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38272,7 +38272,7 @@ class Problem { }"," } }",129.86309999541845,,Completions generated successfully. Ready for display in your IDE.,cdd36d73-b17f-43ad-bc29-f2cee52cfa4c,1,deepseek-ai/deepseek-coder-1.3b-base," return res; -",92,1.0 +",92,1.0 2025-09-22T13:59:54.468167,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38302,7 +38302,7 @@ class Problem { } }",95.31099999730941,,Completions generated successfully. Ready for display in your IDE.,c6ac14f5-3f0c-4220-83b5-3bb6cced4d89,1,deepseek-ai/deepseek-coder-1.3b-base," break; -",72,0.9747 +",72,0.9747 2025-09-22T13:59:54.826083,[1],"//Filter an input array of strings only for ones that start with a given prefix. // >>> filter_by_prefix([], ""a"") // [] @@ -38317,7 +38317,7 @@ function filter_by_prefix(strings, prefix){ } else { return false; } -",336,0.995 +",336,0.995 2025-09-22T13:59:54.923992,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38367,7 +38367,7 @@ class Problem { } }",97.75519999675453,,Completions generated successfully. Ready for display in your IDE.,6012e7df-1a04-4d71-b66a-216efd69358d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T13:59:55.041241,[1],"def digits(n: int) -> int: """"""Given a positive integer n, return the product of the odd digits. Return 0 if all digits are even. @@ -38383,7 +38383,7 @@ class Problem { return digits(d) * (m if m % 2 == 1 else 1) ",117.64750000293134,,Completions generated successfully. Ready for display in your IDE.,fca17bca-7c41-46a9-9918-85a7a65e22f5,1,deepseek-ai/deepseek-coder-1.3b-base," return 0 -",87,0.9286 +",87,0.9286 2025-09-22T13:59:55.176836,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -38424,7 +38424,7 @@ function int_to_mini_roman(number){ } ",135.27520000207005,,Completions generated successfully. Ready for display in your IDE.,80d7d3f0-dae1-4f8b-bb52-e6f6ab91db87,1,deepseek-ai/deepseek-coder-1.3b-base," 9: ""ix"", -",111,1.0 +",111,1.0 2025-09-22T13:59:55.357235,[1],"from typing import List def will_it_fly(q: List[int], w: int) -> bool: @@ -38450,7 +38450,7 @@ def will_it_fly(q: List[int], w: int) -> bool: # 3 is less than the maximum possible weight, and it's balanced. """""" s = sum(q)",,180.14190000394592,,Completions generated successfully. Ready for display in your IDE.,e4f20935-7e15-4690-b599-cddfb3a3bda7,1,deepseek-ai/deepseek-coder-1.3b-base," - return s <= w and s % 2 == 0",156,0.9409 + return s <= w and s % 2 == 0",156,0.9409 2025-09-22T13:59:55.444156,[1],"//You are given a string s. // Your task is to check if the string is hapjs or not. // A string is hapjs if its length is at least 3 and every 3 consecutive letters are distinct @@ -38478,7 +38478,7 @@ function is_happy(s){ }"," return happy; }",86.93649999622721,,Completions generated successfully. Ready for display in your IDE.,768fb591-6799-4dc1-88cb-95ec0911b561,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:55.769356,[1],"//You are given a string representing a sentence, // the sentence contains some words separated by a space, // and you have to return a string that contains the words from the original sentence, @@ -38509,7 +38509,7 @@ function words_in_sentence(sentence){ }",324.52439999906346,,Completions generated successfully. Ready for display in your IDE.,d018c058-2b26-45c8-9140-4deda58ee98c,1,deepseek-ai/deepseek-coder-1.3b-base," else{ for (var i = 3; i < Math.sqrt(number); i += 2){ - ",298,0.8906 + ",298,0.8906 2025-09-22T13:59:56.249988,[1],"//Write a function that returns true if the object q will fly, and false otherwise. // The object q will fly if it's balanced (it is a palindromic array) and the sum of its elements is less than or equal the maximum possible weight w. // Example: @@ -38540,7 +38540,7 @@ function will_it_fly(q, w){ //check to see whether all values are the same in reverse order for(let i = 0; i < Math.floor(q.length/2); i++){ -",456,0.9073 +",456,0.9073 2025-09-22T13:59:56.327020,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38576,7 +38576,7 @@ class Problem { } return count; }",,76.3098000024911,,Completions generated successfully. Ready for display in your IDE.,722d883c-ba2d-4406-835d-1ec13537d669,1,deepseek-ai/deepseek-coder-1.3b-base," -}",46,1.0 +}",46,1.0 2025-09-22T13:59:56.472254,[1],"def correct_bracketing(brackets: str) -> bool: """""" brackets is a string of ""("" and "")"". return True if every opening bracket has a corresponding closing bracket. @@ -38600,7 +38600,7 @@ class Problem { ",145.345199998701,,Completions generated successfully. Ready for display in your IDE.,6a09d0d5-b4d1-48b5-a9ad-c71aff0c7d49,1,deepseek-ai/deepseek-coder-1.3b-base," for char in brackets: -",121,1.0 +",121,1.0 2025-09-22T13:59:56.621305,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -38677,7 +38677,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",148.82650000072317,,Completions generated successfully. Ready for display in your IDE.,95d4188d-38df-4452-9d14-579739e99d5e,1,deepseek-ai/deepseek-coder-1.3b-base," elif a < b: -",119,1.0 +",119,1.0 2025-09-22T13:59:56.747416,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38725,7 +38725,7 @@ class Problem { } }",125.61899999855086,,Completions generated successfully. Ready for display in your IDE.,ed4bd068-822c-4b74-8c3a-f15bf18a648c,1,deepseek-ai/deepseek-coder-1.3b-base," } else { -",98,1.0 +",98,1.0 2025-09-22T13:59:57.006980,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38773,7 +38773,7 @@ class Problem { } }",259.9149000016041,,Completions generated successfully. Ready for display in your IDE.,96c28828-3b2a-48bb-bb5d-b8140da27e4d,1,deepseek-ai/deepseek-coder-1.3b-base," if (l.get(i) % 3 != 0) { -",232,0.9833 +",232,0.9833 2025-09-22T13:59:57.071816,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38795,7 +38795,7 @@ class Problem { BigInteger nSquared = nBigInteger.multiply(nBigInteger); return nSquared.longValue(); }",,64.60409999999683,,Completions generated successfully. Ready for display in your IDE.,1a6ae691-0dc1-406b-9d09-7b01a535bb71,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T13:59:57.180635,[1],"//triples_sum_to_zero takes an array of integers as an input. // it returns true if there are three distinct elements in the array that // sum to zero, and false otherwise. @@ -38823,7 +38823,7 @@ function triples_sum_to_zero(l){ } ",108.53279999719234,,Completions generated successfully. Ready for display in your IDE.,1c927710-5776-41dd-ace8-6924b187f25b,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",83,1.0 +",83,1.0 2025-09-22T13:59:57.771323,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -38864,7 +38864,7 @@ function find_max(words){ //console.log(""Unique: "", JSON.stringify(unique_word)); if(!max_unique || (Object.keys(unique_word).length >= Object.keys(max_unique).length)){ -",563,0.9664 +",563,0.9664 2025-09-22T13:59:57.898155,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38892,7 +38892,7 @@ class Problem { return p; } }",126.71039999986533,,Completions generated successfully. Ready for display in your IDE.,2f157aee-340a-4b75-ace2-cc68e1add9f2,1,deepseek-ai/deepseek-coder-1.3b-base," Long p = 0L; -",102,0.9586 +",102,0.9586 2025-09-22T13:59:58.099530,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -38928,7 +38928,7 @@ class Problem { } }",200.70159999886528,,Completions generated successfully. Ready for display in your IDE.,397961c8-0692-4fd4-ad90-f885cbb7b958,1,deepseek-ai/deepseek-coder-1.3b-base," else n1 = 3 * n1 + 1; -",173,1.0 +",173,1.0 2025-09-22T13:59:58.260837,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -38986,7 +38986,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",161.36939999705646,,Completions generated successfully. Ready for display in your IDE.,7b3f8379-e6de-438e-8ccb-c2ca132ba1b4,1,deepseek-ai/deepseek-coder-1.3b-base," elif x > 1.7: -",131,1.0 +",131,1.0 2025-09-22T13:59:58.379491,[1],"def int_to_mini_roman(number: int) -> str: """""" Given a positive integer, obtain its roman numeral equivalent as a string, @@ -39021,7 +39021,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: roman_numeral += roman_numerals[key] * (number // key) number = number % key"," ",118.67849999543978,,Completions generated successfully. Ready for display in your IDE.,fdef8d23-c07f-4015-b788-466b371e6551,1,deepseek-ai/deepseek-coder-1.3b-base," - return roman_numeral",94,1.0 + return roman_numeral",94,1.0 2025-09-22T13:59:58.477863,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -39053,7 +39053,7 @@ function correct_bracketing(brackets){ } }",98.1541999935871,,Completions generated successfully. Ready for display in your IDE.,fb28d913-889c-4c63-946a-b6e37a338228,1,deepseek-ai/deepseek-coder-1.3b-base," else { -",72,1.0 +",72,1.0 2025-09-22T13:59:58.553259,[1],"//brackets is a string of ""<"" and "">"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""<"") @@ -39079,7 +39079,7 @@ function correct_bracketing(brackets){ } ",74.8576999976649,,Completions generated successfully. Ready for display in your IDE.,e5e8253e-7292-4deb-b46d-f48e240e1a33,1,deepseek-ai/deepseek-coder-1.3b-base," } -",53,1.0 +",53,1.0 2025-09-22T13:59:58.639425,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39108,7 +39108,7 @@ class Problem { } return (sqrt * 2) % p;",},85.95390000118641,,Completions generated successfully. Ready for display in your IDE.,e0a736fb-3cef-4155-93ca-6abfa0f195b8,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T13:59:58.708452,[1],"//Return array of all prefixes from shortest to longest of the input string // >>> all_prefixes(""abc"") // [""a"", ""ab"", ""abc""] @@ -39118,7 +39118,7 @@ function all_prefixes(string){ result.push(string.slice(0, i)); } return result;",,68.86270000541117,,Completions generated successfully. Ready for display in your IDE.,030f2160-fcc0-4ae7-985f-57d38afd0c4f,1,deepseek-ai/deepseek-coder-1.3b-base," -}",47,0.9043 +}",47,0.9043 2025-09-22T13:59:58.789025,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39185,7 +39185,7 @@ class Problem { } } }",80.26969999627909,,Completions generated successfully. Ready for display in your IDE.,520a2cf6-ca7a-48fe-846a-0c32c12008f9,1,deepseek-ai/deepseek-coder-1.3b-base," -",52,1.0 +",52,1.0 2025-09-22T13:59:58.978622,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -39252,7 +39252,7 @@ function numerical_letter_grade(grades){ return grades_array; }",189.20399999478832,,Completions generated successfully. Ready for display in your IDE.,c8185637-7f32-44a3-a06c-8c0fe2220140,1,deepseek-ai/deepseek-coder-1.3b-base," grades_array.push(""A-""); -",160,1.0 +",160,1.0 2025-09-22T13:59:59.031705,[1],"//Input to this function is a string represented multiple groups for nested parentheses separated by spaces. // For each of the group, output the deepest level of nesting of parentheses. // E.g. (()()) has maximum two levels of nesting while ((())) has three. @@ -39281,7 +39281,7 @@ function parse_nested_parens(paren_string){ } ",52.981799999543,,Completions generated successfully. Ready for display in your IDE.,0ad69b2c-6749-4c78-8add-59d66dc927e8,1,deepseek-ai/deepseek-coder-1.3b-base," -",30,1.0 +",30,1.0 2025-09-22T13:59:59.127585,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39304,7 +39304,7 @@ class Problem { } return true;",},96.04029999900376,,Completions generated successfully. Ready for display in your IDE.,3f7025c8-0b27-4666-9b46-dfd81f1ffde7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",72,1.0 +",72,1.0 2025-09-22T13:59:59.212386,[1],"//Return n-th Fibonacci number. // >>> fib(10) // 55 @@ -39319,7 +39319,7 @@ function fib(n){ return fib(n-1) + fib(n-2);","} ",84.14359999733279,,Completions generated successfully. Ready for display in your IDE.,6e3d5137-0190-4b88-97cd-49ccf0f62bac,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T13:59:59.369548,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -39374,7 +39374,7 @@ function by_length(arr){ return arr3; }",156.9078999964404,,Completions generated successfully. Ready for display in your IDE.,d112d7d9-bfec-47d6-9866-55db5158527d,1,deepseek-ai/deepseek-coder-1.3b-base," else if(num == 6){ -",130,1.0 +",130,1.0 2025-09-22T13:59:59.538872,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39416,7 +39416,7 @@ class Problem { } }",169.0870000020368,,Completions generated successfully. Ready for display in your IDE.,1209e292-f6e8-4a0f-8f6a-ce23cc017158,1,deepseek-ai/deepseek-coder-1.3b-base," StringBuilder sb = new StringBuilder(); -",143,1.0 +",143,1.0 2025-09-22T13:59:59.767085,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39484,7 +39484,7 @@ class Problem { } }",228.14970000035828,,Completions generated successfully. Ready for display in your IDE.,e7cca9c5-fe9c-42c8-8fb8-867304c09406,1,deepseek-ai/deepseek-coder-1.3b-base," seen_2_in_a_row = false; -",198,1.0 +",198,1.0 2025-09-22T13:59:59.808907,[1],"//Write a function that takes a message, and encodes in such a // way that it swaps case of all letters, replaces all vowels in // the message with the letter that appears 2 places ahead of that @@ -39506,7 +39506,7 @@ function encode(message){ result += letter.toUpperCase() === letter ? letter.toLowerCase() : letter.toUpperCase(); } return result; -}",41.81139999855077,,Completions generated successfully. Ready for display in your IDE.,ed0ee060-46f9-437f-bd0a-45559ec79371,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 +}",41.81139999855077,,Completions generated successfully. Ready for display in your IDE.,ed0ee060-46f9-437f-bd0a-45559ec79371,1,deepseek-ai/deepseek-coder-1.3b-base,,20,1.0 2025-09-22T14:00:00.091650,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39542,7 +39542,7 @@ class Problem { } }",282.7612999972189,,Completions generated successfully. Ready for display in your IDE.,b7da4953-e399-4b87-8909-9ec8e8abdaf5,1,deepseek-ai/deepseek-coder-1.3b-base," for (int j = 0; j < lst2.size(); j++) { -",255,0.9936 +",255,0.9936 2025-09-22T14:00:00.220161,[1],"from typing import Union, List def split_words(txt: str) -> Union[List[str], int]: @@ -39565,7 +39565,7 @@ def split_words(txt: str) -> Union[List[str], int]: return len([letter for letter in txt if letter in alphabet and alphabet.index(letter) % 2 == 1]) ",127.9385999951046,,Completions generated successfully. Ready for display in your IDE.,aec8d816-759f-49d1-ad84-a3013548b62a,1,deepseek-ai/deepseek-coder-1.3b-base," return txt.split() -",104,1.0 +",104,1.0 2025-09-22T14:00:00.676440,[1],"//You are given an array of integers. // You need to find the largest prime value and return the sum of its digits. // Examples: @@ -39599,7 +39599,7 @@ function skjkasdkd(lst){ for (let i=2; i*i <= n ; ++i ) { if (n % i == 0)return false;} -",431,0.9351 +",431,0.9351 2025-09-22T14:00:01.410197,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -39685,7 +39685,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i if __name__ == ""__main__"": import doctest - doctest",690,0.9727 + doctest",690,0.9727 2025-09-22T14:00:01.489354,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39702,7 +39702,7 @@ class Problem { public static String concatenate(ArrayList strings) { return strings.stream().collect(Collectors.joining());",},79.10419999825535,,Completions generated successfully. Ready for display in your IDE.,abd6e77f-4b61-45f4-a24a-52d4b59163e5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",56,1.0 +",56,1.0 2025-09-22T14:00:01.543164,[1],"//xs represent coefficients of a polynomial. // xs[0] + xs[1] * x + xs[2] * x^2 + .... // Return derivative of this polynomial in the same form. @@ -39716,7 +39716,7 @@ function derivative(xs){ coeff.push(xs[i] * i);"," return coeff; } ",53.55749999580439,,Completions generated successfully. Ready for display in your IDE.,2e3c4590-1f55-4c2e-b4c5-811a74fd8fad,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T14:00:01.663217,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39756,7 +39756,7 @@ class Problem { Collections.sort(b);"," } }",119.9172000051476,,Completions generated successfully. Ready for display in your IDE.,c62c01f9-aa10-4c41-a9d1-6156768e5a27,1,deepseek-ai/deepseek-coder-1.3b-base," return b; -",95,1.0 +",95,1.0 2025-09-22T14:00:01.739254,[1],"//Return a greatest common divisor of two integers a and b // >>> greatest_common_divisor(3, 5) // 1 @@ -39772,7 +39772,7 @@ function greatest_common_divisor(a, b){ ",75.58070000231965,,Completions generated successfully. Ready for display in your IDE.,253108fe-07d0-41bf-b9ec-ef92a100c16c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",51,1.0 +",51,1.0 2025-09-22T14:00:01.832163,[1],"//Write a function vowels_count which takes a string representing // a word as input and returns the number of vowels in the string. // Vowels in this case are 'a', 'e', 'i', 'o', 'u'. Here, 'y' is also a @@ -39796,7 +39796,7 @@ function vowels_count(s){ }","} ",92.78529999573948,,Completions generated successfully. Ready for display in your IDE.,b2f75627-5b14-47b8-b971-a6dfcd4d6d51,1,deepseek-ai/deepseek-coder-1.3b-base," return count; -",69,1.0 +",69,1.0 2025-09-22T14:00:02.030902,[1],"//Given the lengths of the three sides of a triangle. Return the area of // the triangle rounded to 2 decimal points if the three sides form a valid triangle. // Otherwise return -1 @@ -39815,7 +39815,7 @@ function triangle_area(a, b, c){ } ",198.3235000006971,,Completions generated successfully. Ready for display in your IDE.,83e8319e-42f3-4fe0-9c22-2644270525ed,1,deepseek-ai/deepseek-coder-1.3b-base," let s= (a + b + c)/2; -",176,0.9288 +",176,0.9288 2025-09-22T14:00:02.178530,[1],"def largest_prime_factor(n: int) -> int: """"""Return the largest prime factor of n. Assume n > 1 and is not a prime. >>> largest_prime_factor(13195) @@ -39833,7 +39833,7 @@ function triangle_area(a, b, c){ ",147.68890000414103,,Completions generated successfully. Ready for display in your IDE.,f0735df7-3104-4ecd-91c5-109a155fd96b,1,deepseek-ai/deepseek-coder-1.3b-base," while (n != 1): -",124,0.9375 +",124,0.9375 2025-09-22T14:00:02.664807,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39865,7 +39865,7 @@ class Problem { } }",486.18010000063805,,Completions generated successfully. Ready for display in your IDE.,39c13816-b2ae-4c05-8949-68bfe586467e,1,deepseek-ai/deepseek-coder-1.3b-base," if ((guess.get(i).equals(game.get(i))) || Math.abs(guess.get(i)).equals(game.get(i))) { -",458,0.9841 +",458,0.9841 2025-09-22T14:00:02.829619,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -39916,7 +39916,7 @@ class Problem { } }",164.3882999997004,,Completions generated successfully. Ready for display in your IDE.,2ebb46ef-28e5-4a97-9a1b-364707a66380,1,deepseek-ai/deepseek-coder-1.3b-base," if (count > 3) { -",131,1.0 +",131,1.0 2025-09-22T14:00:03.530781,[1],"//Complete the function that takes two integers and returns // the product of their unit digits. // Assume the input is always valid. @@ -39934,7 +39934,7 @@ function multiply(a, b){ let bStr = b.toString()","} ",700.7826999979443,,Completions generated successfully. Ready for display in your IDE.,2da7fdc6-02b1-423d-b54c-aa784545221b,1,deepseek-ai/deepseek-coder-1.3b-base," - return parseInt((aStr[aStr.length-1] * bStr[bStr.length-1])) + (parseInt(aStr[(aStr.length)-2])*parseInt(bStr[(bStr.length)-2]))+multiply(Math",668,0.9396 + return parseInt((aStr[aStr.length-1] * bStr[bStr.length-1])) + (parseInt(aStr[(aStr.length)-2])*parseInt(bStr[(bStr.length)-2]))+multiply(Math",668,0.9396 2025-09-22T14:00:03.633936,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -40001,7 +40001,7 @@ function numerical_letter_grade(grades){ return grades_array; }",102.83550000167452,,Completions generated successfully. Ready for display in your IDE.,13dd8483-b7db-4ba5-a3d0-07d0b29a4cbb,1,deepseek-ai/deepseek-coder-1.3b-base," } -",76,1.0 +",76,1.0 2025-09-22T14:00:03.761353,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -40037,7 +40037,7 @@ function numerical_letter_grade(grades){ ",127.2949999984121,,Completions generated successfully. Ready for display in your IDE.,3169d4ae-8d81-4fa7-a345-f5e82b3de3ea,1,deepseek-ai/deepseek-coder-1.3b-base," return 'No' -",99,1.0 +",99,1.0 2025-09-22T14:00:03.818794,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40071,7 +40071,7 @@ class Problem { return true; } }",57.42439999448834,,Completions generated successfully. Ready for display in your IDE.,e1f112d1-2d5c-4170-9683-feb2c2b30876,1,deepseek-ai/deepseek-coder-1.3b-base," -",34,1.0 +",34,1.0 2025-09-22T14:00:03.914726,[1],"def encode(message: str) -> str: """""" Write a function that takes a message, and encodes in such a @@ -40098,7 +40098,7 @@ class Problem { return ''.join(map(lambda ch: encode_vowel(swap_case(ch)), message)) ",95.70849999727216,,Completions generated successfully. Ready for display in your IDE.,1e9c909d-f75f-4425-88e6-894269396143,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",68,0.9167 +",68,0.9167 2025-09-22T14:00:04.214127,[1],"//You are given a string s. // if s[i] is a letter, reverse its case from lower to upper or vise versa, // otherwise keep it as it is. @@ -40127,7 +40127,7 @@ function solve(s){ ",298.90890000388026,,Completions generated successfully. Ready for display in your IDE.,aa06335f-426e-4894-bd39-351f2723167f,1,deepseek-ai/deepseek-coder-1.3b-base," // console.log({letter: char}); return (/\d/.test(char)) - ",269,0.9457 + ",269,0.9457 2025-09-22T14:00:04.497926,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40170,7 +40170,7 @@ class Problem { } }",283.84270000242395,,Completions generated successfully. Ready for display in your IDE.,4ad88a01-603b-4bbd-8f5f-4f7bd2ec4b2c,1,deepseek-ai/deepseek-coder-1.3b-base," long minX = ((n - 3 * d) / 2) / 2; -",260,0.9549 +",260,0.9549 2025-09-22T14:00:04.593385,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40214,7 +40214,7 @@ class Problem { } }",95.11860000202432,,Completions generated successfully. Ready for display in your IDE.,13dff80a-ee2a-42b4-b213-c25140d19ba2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:04.685771,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40246,7 +40246,7 @@ class Problem { } }",91.92819999589119,,Completions generated successfully. Ready for display in your IDE.,b1975303-85b6-4e51-aa36-f23b83f9db12,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:04.850026,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -40304,7 +40304,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",164.48269999818876,,Completions generated successfully. Ready for display in your IDE.,c3d5f30d-e201-4934-b8e6-88559bfa2896,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('C+') -",131,1.0 +",131,1.0 2025-09-22T14:00:05.077136,[1],"//sum_to_n is a function that sums numbers from 1 to n. // >>> sum_to_n(30) // 465 @@ -40323,7 +40323,7 @@ function sum_to_n(n){ } ",226.75609999714652,,Completions generated successfully. Ready for display in your IDE.,fb26efae-6a9a-4edf-af39-88c723a5696b,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=1;i<=n;++i){ -",205,1.0 +",205,1.0 2025-09-22T14:00:05.223297,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -40400,7 +40400,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",145.87639999808744,,Completions generated successfully. Ready for display in your IDE.,36d01ddb-a519-4096-bb78-0261f99a84a5,1,deepseek-ai/deepseek-coder-1.3b-base," elif a < b: -",119,1.0 +",119,1.0 2025-09-22T14:00:05.332487,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -40438,7 +40438,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",109.0669999975944,,Completions generated successfully. Ready for display in your IDE.,3bfb80f1-2d17-413f-ad80-b52addbfdcee,1,deepseek-ai/deepseek-coder-1.3b-base," }; -",69,1.0 +",69,1.0 2025-09-22T14:00:05.401890,[1],"//Write a function that accepts an array of strings. // The array contains different words. Return the word with maximum number // of unique characters. If multiple strings have maximum number of unique @@ -40476,7 +40476,7 @@ function find_max(words){ } return max_unique_word; }",69.32479999522911,,Completions generated successfully. Ready for display in your IDE.,da42568d-8f6b-433e-81a5-2db89d825833,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T14:00:05.497749,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40520,7 +40520,7 @@ class Problem { } }",95.04990000277758,,Completions generated successfully. Ready for display in your IDE.,26f9504d-f5dd-430d-9606-d8e6f62ebdcf,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T14:00:05.666896,[1],"//Given a string of words, return an array of words split on whitespace, if no whitespaces exists in the text you // should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the // alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 @@ -40549,7 +40549,7 @@ function split_words(txt){ } }",168.89660000015283,,Completions generated successfully. Ready for display in your IDE.,f8253f01-a609-42bf-a277-1b56e34ac4fa,1,deepseek-ai/deepseek-coder-1.3b-base," else if (txt.includes(',')){ -",145,1.0 +",145,1.0 2025-09-22T14:00:06.038811,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -40582,7 +40582,7 @@ function same_chars(s0, s1){ //check reverse of string 2 for(let i = 0; i < s1.length; i++){ -",340,0.9455 +",340,0.9455 2025-09-22T14:00:06.122974,[1],"//pairs_sum_to_zero takes an array of integers as an input. // it returns true if there are two distinct elements in the array that // sum to zero, and false otherwise. @@ -40610,7 +40610,7 @@ function pairs_sum_to_zero(l){ } ",83.6669000054826,,Completions generated successfully. Ready for display in your IDE.,8a5171f3-53a4-43b4-bcb6-b5d003122824,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T14:00:06.233027,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -40652,7 +40652,7 @@ function compare_one(a, b){ } ",109.72860000038054,,Completions generated successfully. Ready for display in your IDE.,8719a6b1-3173-4ab4-8fb7-685b42f8bbfd,1,deepseek-ai/deepseek-coder-1.3b-base," return b; -",85,1.0 +",85,1.0 2025-09-22T14:00:06.297903,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -40684,7 +40684,7 @@ function correct_bracketing(brackets){ else { return false; }",,64.4006000002264,,Completions generated successfully. Ready for display in your IDE.,439ec4f4-bb19-452a-85e5-1aa73a56fc20,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T14:00:06.737045,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40730,7 +40730,7 @@ class Problem { }",439.50520000362303,,Completions generated successfully. Ready for display in your IDE.,4dea9f4b-9294-44a3-a5d4-56e07d1e5611,1,deepseek-ai/deepseek-coder-1.3b-base," int j = q.size() - i - 1; if (!Objects.equals(q.get(i), q.get(j))) { -",411,0.9726 +",411,0.9726 2025-09-22T14:00:06.828793,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -40774,7 +40774,7 @@ function histogram(test){ } ",91.56369999982417,,Completions generated successfully. Ready for display in your IDE.,a349120b-52de-428e-bdb4-fe3dc220d388,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:06.940128,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40794,7 +40794,7 @@ class Problem { } }",111.15060000156518,,Completions generated successfully. Ready for display in your IDE.,7c6cccc5-39d1-47e8-b9ae-edab1568dbc8,1,deepseek-ai/deepseek-coder-1.3b-base," return i; -",82,1.0 +",82,1.0 2025-09-22T14:00:07.072638,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40832,7 +40832,7 @@ class Problem { } }",131.58280000061495,,Completions generated successfully. Ready for display in your IDE.,16d53a97-252b-4da9-b6f9-83559c592465,1,deepseek-ai/deepseek-coder-1.3b-base," } -",76,1.0 +",76,1.0 2025-09-22T14:00:07.242094,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40910,7 +40910,7 @@ class Problem { } }",169.72979999991367,,Completions generated successfully. Ready for display in your IDE.,26413408-0778-458c-94d1-c32eb6b30482,1,deepseek-ai/deepseek-coder-1.3b-base," while (num >= 10) { -",142,1.0 +",142,1.0 2025-09-22T14:00:07.339703,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -40955,7 +40955,7 @@ class Problem { } }",97.10500000073807,,Completions generated successfully. Ready for display in your IDE.,21df2558-0b6f-43be-b8d4-7f01f1804539,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T14:00:07.547082,[1],"from typing import List def minPath(grid: List[List[int]], k: int) -> List[int]: @@ -41003,7 +41003,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: path.append(min_val) return path",207.42329999484355,,Completions generated successfully. Ready for display in your IDE.,1fd17019-cafd-47a4-b208-d434171d066f,1,deepseek-ai/deepseek-coder-1.3b-base," min_val = grid[i][j] -",171,1.0 +",171,1.0 2025-09-22T14:00:07.602348,[1],"def triangle_area(a: int, b: int, c: int) -> float: """""" Given the lengths of the three sides of a triangle. Return the area of @@ -41023,7 +41023,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: return -1 ",54.86689999816008,,Completions generated successfully. Ready for display in your IDE.,ac90d796-c5c5-4a5b-aa4d-495945d47542,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T14:00:07.842897,[1],"//This function takes an array l and returns an array l' such that // l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal // to the values of the corresponding indicies of l, but sorted. @@ -41053,7 +41053,7 @@ function sort_third(l){ return x; }",240.79920000076527,,Completions generated successfully. Ready for display in your IDE.,5e8efd96-96cd-4352-89d7-3f2369068c0b,1,deepseek-ai/deepseek-coder-1.3b-base," let y=new Array(Math.floor((x.length)/3)); -",217,1.0 +",217,1.0 2025-09-22T14:00:07.927182,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -41085,7 +41085,7 @@ function correct_bracketing(brackets){ else { return false;",},84.03840000391938,,Completions generated successfully. Ready for display in your IDE.,2a85a275-f122-413c-aadf-3527163fcd75,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T14:00:08.379638,[1],"//This function takes two positive numbers x and y and returns the // biggest even integer number that is in the range [x, y] inclusive. If // there's no such number, then the function should return -1. @@ -41103,7 +41103,7 @@ function choose_num(x, y){ } ",451.9014999968931,,Completions generated successfully. Ready for display in your IDE.,505a97de-6b58-4b3c-864d-b1ff4a335ca3,1,deepseek-ai/deepseek-coder-1.3b-base," if (i % 2 == 0 && Math.floor(Math.log10(i)) == Math.ceil(Math.log10(i))) { -",411,0.9384 +",411,0.9384 2025-09-22T14:00:08.472158,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41142,7 +41142,7 @@ class Problem { } }",92.51889999723062,,Completions generated successfully. Ready for display in your IDE.,4134a6dc-2fcb-4bb9-bfa9-0c3c389b1f4e,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:08.839805,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41178,7 +41178,7 @@ class Problem { return """"; } StringBuilder sb = new StringBuilder(); -",342,0.9473 +",342,0.9473 2025-09-22T14:00:08.968743,[1],"from typing import List def minPath(grid: List[List[int]], k: int) -> List[int]: @@ -41226,7 +41226,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: path.append(min_val) return path",128.1656999999541,,Completions generated successfully. Ready for display in your IDE.,52e61270-1096-4b77-b39c-8aee6a35ca82,1,deepseek-ai/deepseek-coder-1.3b-base," while True: -",98,1.0 +",98,1.0 2025-09-22T14:00:09.063447,[1],"def is_happy(s: str) -> bool: """"""You are given a string s. Your task is to check if the string is happy or not. @@ -41253,7 +41253,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: if s[i] == s[i + 1] or s[i] == s[i + 2] or s[i + 1] == s[i + 2]: return False"," ",94.7870999953011,,Completions generated successfully. Ready for display in your IDE.,b4aed541-dee8-44f6-9b04-0d8985081fb8,1,deepseek-ai/deepseek-coder-1.3b-base," - return True",67,1.0 + return True",67,1.0 2025-09-22T14:00:09.178203,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -41325,7 +41325,7 @@ function minPath(grid, k){ return path; }",114.7075999979279,,Completions generated successfully. Ready for display in your IDE.,fdf9ce3d-9eb1-40c2-beb2-3cfa8b9a1cbc,1,deepseek-ai/deepseek-coder-1.3b-base," } -",83,1.0 +",83,1.0 2025-09-22T14:00:09.268418,[1],"//The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fib4(0) -> 0 // fib4(1) -> 0 @@ -41353,7 +41353,7 @@ function fib4(n){ } ",89.83269999589538,,Completions generated successfully. Ready for display in your IDE.,eb5d118b-61dd-4640-bc42-de6ec8e074fa,1,deepseek-ai/deepseek-coder-1.3b-base," } -",66,1.0 +",66,1.0 2025-09-22T14:00:09.463944,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -41395,7 +41395,7 @@ function compare_one(a, b){ } ",195.19620000210125,,Completions generated successfully. Ready for display in your IDE.,b770a012-b94e-49e3-b63f-5a86e92d7215,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (a[0] < b){ -",170,1.0 +",170,1.0 2025-09-22T14:00:09.705211,[1],"from typing import List def do_algebra(operator: List[str], operand: List[int]) -> int: @@ -41430,7 +41430,7 @@ def do_algebra(operator: List[str], operand: List[int]) -> int: return eval("""".join(expr))",241.15200000233017,,Completions generated successfully. Ready for display in your IDE.,be789a8d-63ba-464b-a007-4ce7dd656eb3,1,deepseek-ai/deepseek-coder-1.3b-base," if op == ""+"": expr.append(""+"") -",214,0.9931 +",214,0.9931 2025-09-22T14:00:09.791196,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41456,7 +41456,7 @@ class Problem { } return levels;",},86.04529999865917,,Completions generated successfully. Ready for display in your IDE.,c7a5bc2f-ca22-4514-b627-fb0f139c5ced,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:09.899777,[1],"def car_race_collision(n: int) -> int: """""" Imagine a road that's a perfectly straight infinitely long line. @@ -41477,7 +41477,7 @@ class Problem { for j in range(l): if left_cars[i] == right_cars[j]:", return count,108.29830000147922,,Completions generated successfully. Ready for display in your IDE.,384918b1-2179-4f85-ab9b-acc9ca8b2bec,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1 -",82,1.0 +",82,1.0 2025-09-22T14:00:09.989336,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41508,7 +41508,7 @@ class Problem { } }",89.46520000608871,,Completions generated successfully. Ready for display in your IDE.,d98ad3a3-85fc-4c21-9522-e1ccad046b2a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:10.033153,[1],"def fibfib(n: int) -> int: """"""The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: fibfib(0) == 0 @@ -41528,7 +41528,7 @@ class Problem { n -= 1 return a -",43.17379999702098,,Completions generated successfully. Ready for display in your IDE.,12316f0d-c5e5-470e-b32b-66272edc85d1,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",43.17379999702098,,Completions generated successfully. Ready for display in your IDE.,12316f0d-c5e5-470e-b32b-66272edc85d1,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T14:00:10.096162,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41569,7 +41569,7 @@ class Problem { return sb.toString(); } }",62.8045999983442,,Completions generated successfully. Ready for display in your IDE.,a0131e6a-4e62-4cdc-9d3c-7ac2746dcb32,1,deepseek-ai/deepseek-coder-1.3b-base," -",39,1.0 +",39,1.0 2025-09-22T14:00:10.235954,[1],"from typing import List, Tuple, Optional def largest_smallest_integers(lst: List[int]) -> Tuple[Optional[int], Optional[int]]: @@ -41597,7 +41597,7 @@ def largest_smallest_integers(lst: List[int]) -> Tuple[Optional[int], Optional[i if largest_negative is None or num > largest_negative:"," return (largest_negative, smallest_positive) ",140.07939999282826,,Completions generated successfully. Ready for display in your IDE.,fa8453ad-e19b-4a7e-9316-be3d165c9b7d,1,deepseek-ai/deepseek-coder-1.3b-base," largest_negative = num -",104,1.0 +",104,1.0 2025-09-22T14:00:10.317381,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41617,7 +41617,7 @@ class Problem { } }",81.14039999782108,,Completions generated successfully. Ready for display in your IDE.,096ab786-450d-4947-853c-f1e3d7780109,1,deepseek-ai/deepseek-coder-1.3b-base," } -",60,1.0 +",60,1.0 2025-09-22T14:00:10.423719,[1],"//You're given an array of deposit and withdrawal operations on a bank account that starts with // zero balance. Your task is to detect if at any point the balance of account fallls below zero, and // at that point function should return true. Otherwise it should return false. @@ -41635,7 +41635,7 @@ function below_zero(operations){ } ",106.11229999631178,,Completions generated successfully. Ready for display in your IDE.,b8afb90b-6375-4d4d-8b4e-3becc79872b0,1,deepseek-ai/deepseek-coder-1.3b-base," return true; -",83,1.0 +",83,1.0 2025-09-22T14:00:10.598744,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -41707,7 +41707,7 @@ function minPath(grid, k){ return path; }",175.0746000034269,,Completions generated successfully. Ready for display in your IDE.,e9bd0666-dac5-4afa-9949-d99e4a080f32,1,deepseek-ai/deepseek-coder-1.3b-base," path.push(startCell); -",146,1.0 +",146,1.0 2025-09-22T14:00:10.750058,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -41775,7 +41775,7 @@ function do_algebra(operator, operand){ } ",150.9479000014835,,Completions generated successfully. Ready for display in your IDE.,b58ddf50-5109-4a35-9b89-75c26e49d0f1,1,deepseek-ai/deepseek-coder-1.3b-base," do_again = true; -",123,1.0 +",123,1.0 2025-09-22T14:00:10.979306,[1],"//Write a function that accepts an array of strings as a parameter, // deletes the strings that have odd lengths from it, // and returns the resulted array with a sorted order, @@ -41819,7 +41819,7 @@ function sorted_list_sum(lst){ }); } var a = [""aa"", ""a"", ""aaa""];",,228.8929000060307,,Completions generated successfully. Ready for display in your IDE.,d17cf00c-21a6-4b9d-85c0-36ca72c845c0,1,deepseek-ai/deepseek-coder-1.3b-base," -console.log(""Result: "",sorted_list_sum(a));",203,0.9218 +console.log(""Result: "",sorted_list_sum(a));",203,0.9218 2025-09-22T14:00:11.065655,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41836,7 +41836,7 @@ class Problem { public static long strlen(String string) { return string.length();",},86.27139999589417,,Completions generated successfully. Ready for display in your IDE.,4c9083e2-c0d4-4da3-bd95-5fa294c747cd,1,deepseek-ai/deepseek-coder-1.3b-base," } -",65,1.0 +",65,1.0 2025-09-22T14:00:11.148950,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -41873,7 +41873,7 @@ function is_prime(n){ return true; } ",83.07590000185883,,Completions generated successfully. Ready for display in your IDE.,cad2257e-2f8f-4536-a84e-2196caa7d952,1,deepseek-ai/deepseek-coder-1.3b-base," - }",60,1.0 + }",60,1.0 2025-09-22T14:00:11.392927,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -41913,7 +41913,7 @@ function match_parens(lst){ return 'No'; }",243.87739999656333,,Completions generated successfully. Ready for display in your IDE.,4732aacf-e0d1-4a69-b214-2faa8beb0724,1,deepseek-ai/deepseek-coder-1.3b-base," if (!stack || !stack.slice(-1)[0] == open){ -",219,0.9911 +",219,0.9911 2025-09-22T14:00:11.758385,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41944,7 +41944,7 @@ class Problem { } }",365.48929999844404,,Completions generated successfully. Ready for display in your IDE.,691e5e34-216a-4a22-8bb8-9291de0fd7b9,1,deepseek-ai/deepseek-coder-1.3b-base," if ((result.charAt(i) == '_') && (result.charAt(i+1) == ' ')) { -",341,0.9723 +",341,0.9723 2025-09-22T14:00:11.831946,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -41991,7 +41991,7 @@ class Problem { return l_s; } }",72.47119999374263,,Completions generated successfully. Ready for display in your IDE.,62045d30-a3ca-424e-b466-f1e2332e0581,1,deepseek-ai/deepseek-coder-1.3b-base," -",46,1.0 +",46,1.0 2025-09-22T14:00:12.525842,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42024,7 +42024,7 @@ class Problem { memo[0] = 0; memo[1] = 0; memo[2] = 2; - memo[",670,0.9273 + memo[",670,0.9273 2025-09-22T14:00:12.696771,[1],"//Given an array of positive integers x. return a sorted array of all // elements that hasn't any even digit. // Note: Returned array should be sorted in increasing order. @@ -42044,7 +42044,7 @@ function unique_digits(x){ ",170.80469999928027,,Completions generated successfully. Ready for display in your IDE.,ea038172-cc39-48c4-a04a-d9abb5988526,1,deepseek-ai/deepseek-coder-1.3b-base," result.push(Number(num)) -",134,0.9844 +",134,0.9844 2025-09-22T14:00:12.751946,[1],"//Write a function that takes an array of numbers as input and returns // the number of elements in the array that are greater than 10 and both // first and last digits of a number are odd (1, 3, 5, 7, 9). @@ -42064,7 +42064,7 @@ function specialFilter(nums){ } return count; } -",54.95290000544628,,Completions generated successfully. Ready for display in your IDE.,cf34aa1c-396f-4a64-b0b6-7fca8aeaafb8,1,deepseek-ai/deepseek-coder-1.3b-base, ,31,0.8564 +",54.95290000544628,,Completions generated successfully. Ready for display in your IDE.,cf34aa1c-396f-4a64-b0b6-7fca8aeaafb8,1,deepseek-ai/deepseek-coder-1.3b-base, ,31,0.8564 2025-09-22T14:00:12.907961,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42099,7 +42099,7 @@ class Problem { } }",155.51330000016605,,Completions generated successfully. Ready for display in your IDE.,2f734107-8e86-47f4-8112-6bf587df742a,1,deepseek-ai/deepseek-coder-1.3b-base," long ans = 0L; -",129,1.0 +",129,1.0 2025-09-22T14:00:13.067127,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42120,7 +42120,7 @@ class Problem { } }",159.1855000006035,,Completions generated successfully. Ready for display in your IDE.,0f1b754b-49dc-4966-8fb6-91a883178a02,1,deepseek-ai/deepseek-coder-1.3b-base," result += Long.toString(i); -",134,1.0 +",134,1.0 2025-09-22T14:00:13.109395,[1],"//Find how many times a given substring can be found in the original string. Count overlaping cases. // >>> how_many_times("""", ""a"") // 0 @@ -42135,7 +42135,7 @@ function how_many_times(string, substring){ i = string.indexOf(substring, i) + 1; } return count; -}",41.99769999831915,,Completions generated successfully. Ready for display in your IDE.,2cfd41ac-1083-444c-949b-a7d2b0fe8017,1,deepseek-ai/deepseek-coder-1.3b-base,,20,0.5 +}",41.99769999831915,,Completions generated successfully. Ready for display in your IDE.,2cfd41ac-1083-444c-949b-a7d2b0fe8017,1,deepseek-ai/deepseek-coder-1.3b-base,,20,0.5 2025-09-22T14:00:13.192606,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42152,7 +42152,7 @@ class Problem { public static long add(long x, long y) { return x + y;",},83.22790000238456,,Completions generated successfully. Ready for display in your IDE.,75f30728-a6be-4192-8ba9-291901a29512,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:13.285737,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -42201,7 +42201,7 @@ function check_dict_case(dict){ return true; }",92.90850000252249,,Completions generated successfully. Ready for display in your IDE.,507b9a85-b25e-4436-a0c7-27244834871b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",67,1.0 +",67,1.0 2025-09-22T14:00:13.412155,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42269,7 +42269,7 @@ class Problem { } }",125.97100000130013,,Completions generated successfully. Ready for display in your IDE.,be465573-4a31-4a3d-a2fc-e28bf9d72fe3,1,deepseek-ai/deepseek-coder-1.3b-base," else { -",94,1.0 +",94,1.0 2025-09-22T14:00:13.543104,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -42337,7 +42337,7 @@ function do_algebra(operator, operand){ } ",130.39450000360375,,Completions generated successfully. Ready for display in your IDE.,3bb01e86-01f4-4130-a3cf-d83c31825534,1,deepseek-ai/deepseek-coder-1.3b-base," count = 0; -",103,1.0 +",103,1.0 2025-09-22T14:00:13.630201,[1],"//You are given a non-empty array of positive integers. Return the greatest integer that is greater than // zero, and has a frequency greater than or equal to the value of the integer itself. // The frequency of an integer is the number of times it appears in the array. @@ -42372,7 +42372,7 @@ function search(lst){ } ",87.28919999703066,,Completions generated successfully. Ready for display in your IDE.,5c93da8c-2164-4d0d-8469-9690fae782d7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:13.723542,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42407,7 +42407,7 @@ class Problem { } }",92.99450000253273,,Completions generated successfully. Ready for display in your IDE.,40afe20f-f951-4848-bee0-303ab6cf0d7f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:13.801593,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42430,7 +42430,7 @@ class Problem { } return ans; }",,77.6112000021385,,Completions generated successfully. Ready for display in your IDE.,829d1eb5-69be-4a07-aa13-a53736dbadaf,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:00:13.861962,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42468,7 +42468,7 @@ class Problem { return res; } }",60.07429999590386,,Completions generated successfully. Ready for display in your IDE.,73c7c3c5-7fea-4069-8acd-825027b93807,1,deepseek-ai/deepseek-coder-1.3b-base," -",36,1.0 +",36,1.0 2025-09-22T14:00:14.020929,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42546,7 +42546,7 @@ class Problem { } }",159.32080000493443,,Completions generated successfully. Ready for display in your IDE.,5f7724b1-2e85-494e-9b3e-36c18270fcf4,1,deepseek-ai/deepseek-coder-1.3b-base," num -= 500; -",133,1.0 +",133,1.0 2025-09-22T14:00:14.311494,[1],"//Create a function that takes a value (string) representing a number // and returns the closest integer to it. If the number is equidistant // from two integers, round it away from zero. @@ -42576,7 +42576,7 @@ function closest_integer(value){ } ",290.10489999927813,,Completions generated successfully. Ready for display in your IDE.,852e78cc-c96c-4b25-abf0-1e1948d8b434,1,deepseek-ai/deepseek-coder-1.3b-base," if(Math.abs(-value + 0.5) < Math.abs((+value))){ -",266,0.925 +",266,0.925 2025-09-22T14:00:14.404371,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42613,7 +42613,7 @@ class Problem { } }",92.8801999980351,,Completions generated successfully. Ready for display in your IDE.,b51fca1a-2b13-4924-ad88-b751c3d45edc,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:14.467402,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42652,7 +42652,7 @@ class Problem { return words.get(0); } }",62.201300002925564,,Completions generated successfully. Ready for display in your IDE.,6cb44fc5-59a7-4b28-8e5f-32fc026e9fd5,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T14:00:14.595913,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42682,7 +42682,7 @@ class Problem { } }",129.12320000032196,,Completions generated successfully. Ready for display in your IDE.,df21c070-c54b-4439-b390-bebf8b9d08d9,1,deepseek-ai/deepseek-coder-1.3b-base," n /= d; -",105,1.0 +",105,1.0 2025-09-22T14:00:14.916038,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -42721,7 +42721,7 @@ function max_fill(grid, capacity){ //loop through rows for (var i = 0; i < r; i++ ){ -",295,0.9972 +",295,0.9972 2025-09-22T14:00:15.191945,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42757,7 +42757,7 @@ class Problem { } }",275.5455000005895,,Completions generated successfully. Ready for display in your IDE.,fbde3beb-d17c-40a2-9167-dd4d24caac61,1,deepseek-ai/deepseek-coder-1.3b-base," List ans = Arrays.asList(odds_arr); -",245,1.0 +",245,1.0 2025-09-22T14:00:15.316371,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42796,7 +42796,7 @@ class Problem { return result; } }",124.30180000228574,,Completions generated successfully. Ready for display in your IDE.,7364a3dd-c6d5-414b-bf7b-87c9a9b3bfdc,1,deepseek-ai/deepseek-coder-1.3b-base," int result = 0; -",100,1.0 +",100,1.0 2025-09-22T14:00:15.589100,[1],"def encode(message: str) -> str: """""" Write a function that takes a message, and encodes in such a @@ -42825,7 +42825,7 @@ class Problem { elif ch == ' ': return '_' raise ValueError(""Invalid character"") -",248,0.9611 +",248,0.9611 2025-09-22T14:00:15.661508,[1],"//Write a function that takes a string and returns true if the string // length is a prime number or false otherwise // Examples @@ -42845,7 +42845,7 @@ function prime_length(string){ } else { return false; }",,72.19650000479305,,Completions generated successfully. Ready for display in your IDE.,4ad07b3d-1a84-47a7-9a46-fa71b6610d41,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T14:00:15.748508,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42879,7 +42879,7 @@ class Problem { } }",87.1329000001424,,Completions generated successfully. Ready for display in your IDE.,4559fc9d-494f-49b0-8452-ecc723fa6b39,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:16.029934,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42901,7 +42901,7 @@ class Problem { } }",281.336599997303,,Completions generated successfully. Ready for display in your IDE.,30829026-b93a-439d-8031-6e9e5e8111dc,1,deepseek-ai/deepseek-coder-1.3b-base," BigInteger nSquared = (nBigInteger).multiply((nBigInteger)); -",258,0.946 +",258,0.946 2025-09-22T14:00:16.126545,[1],"def get_closest_vowel(word: str) -> str: """"""You are given a word. Your task is to find the closest vowel that stands between two consonants from the right side of the word (case sensitive). @@ -42927,7 +42927,7 @@ class Problem { return '' ",96.13520000129938,,Completions generated successfully. Ready for display in your IDE.,d0f64d1e-a025-4ffa-92d4-02c6030a0965,1,deepseek-ai/deepseek-coder-1.3b-base," return '' -",72,1.0 +",72,1.0 2025-09-22T14:00:16.356997,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -42971,7 +42971,7 @@ class Problem { } }",229.75200000655605,,Completions generated successfully. Ready for display in your IDE.,c434b741-7e6e-4f60-9ac1-af02b1e50eee,1,deepseek-ai/deepseek-coder-1.3b-base," long firstDigit = h.getFirstDigit(x); -",205,1.0 +",205,1.0 2025-09-22T14:00:16.465034,[1],"from typing import List def match_parens(lst: List[str]) -> str: @@ -43006,7 +43006,7 @@ def match_parens(lst: List[str]) -> str: close_parens += 1 if open_parens == 0 and close_parens == 0: return 'Yes'",,108.11709999688901,,Completions generated successfully. Ready for display in your IDE.,a59e286a-edf2-4c44-b60c-5e3eb5529c46,1,deepseek-ai/deepseek-coder-1.3b-base," - return 'No'",84,0.965 + return 'No'",84,0.965 2025-09-22T14:00:16.644342,[1],"from typing import Dict def histogram(test: str) -> Dict[str, int]: @@ -43038,7 +43038,7 @@ def histogram(test: str) -> Dict[str, int]: return result ",178.9883000019472,,Completions generated successfully. Ready for display in your IDE.,356dcef8-bac6-4263-8216-3d673d950fa3,1,deepseek-ai/deepseek-coder-1.3b-base," if letters[letter] == most_count: -",155,1.0 +",155,1.0 2025-09-22T14:00:16.825095,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43079,7 +43079,7 @@ class Problem { }",180.2112000004854,,Completions generated successfully. Ready for display in your IDE.,e6c2f335-780c-4f5e-a3f1-b086881bb563,1,deepseek-ai/deepseek-coder-1.3b-base," else { return compareResult; } -",151,1.0 +",151,1.0 2025-09-22T14:00:16.905908,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43111,7 +43111,7 @@ class Problem { } return false; }",,81.18430000467924,,Completions generated successfully. Ready for display in your IDE.,0fa8cd08-ea60-4b74-a065-29a0b35ab564,1,deepseek-ai/deepseek-coder-1.3b-base," -}",47,1.0 +}",47,1.0 2025-09-22T14:00:16.972623,[1],"//Check if in given array of numbers, are any two numbers closer to each other than // given threshold. // >>> has_close_elements([1.0, 2.0, 3.0], 0.5) @@ -43127,7 +43127,7 @@ function has_close_elements(numbers, threshold){ } } return false;",,66.36520000029122,,Completions generated successfully. Ready for display in your IDE.,537d1c0b-4aff-4f27-abd7-db5a327d647d,1,deepseek-ai/deepseek-coder-1.3b-base," -}",44,1.0 +}",44,1.0 2025-09-22T14:00:17.081879,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43152,7 +43152,7 @@ class Problem { result = set.size();"," } }",108.97020000265911,,Completions generated successfully. Ready for display in your IDE.,02f08fb6-e974-41b3-9c34-033bc7b84b27,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",83,1.0 +",83,1.0 2025-09-22T14:00:17.179195,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43203,7 +43203,7 @@ class Problem { } }",96.31419999641366,,Completions generated successfully. Ready for display in your IDE.,1eb7eb54-781c-4b70-87e5-27db8b1dfba4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:17.503262,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43230,7 +43230,7 @@ class Problem { return -1; } }",324.5335999963572,,Completions generated successfully. Ready for display in your IDE.,8d38e1e5-f582-4a97-9c8d-c4b186291beb,1,deepseek-ai/deepseek-coder-1.3b-base," if ((a < b + c) && (b < a + c) && (c < a + b)) { -",299,0.9573 +",299,0.9573 2025-09-22T14:00:17.757252,[1],"//Write a function count_nums which takes an array of integers and returns // the number of elements which has a sum of digits > 0. // If a number is negative, then its first signed digit will be negative: @@ -43264,7 +43264,7 @@ function count_nums(arr){ return count; }",253.74149999697693,,Completions generated successfully. Ready for display in your IDE.,6f648d6e-ecec-4647-8cf5-408df0ed5eb3,1,deepseek-ai/deepseek-coder-1.3b-base," for(var k = 0; k < str.length; k++) { -",230,1.0 +",230,1.0 2025-09-22T14:00:17.893210,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -43300,7 +43300,7 @@ function count_nums(arr){ ",135.65949999610893,,Completions generated successfully. Ready for display in your IDE.,0071fd05-2108-422a-8308-c96288e3a84a,1,deepseek-ai/deepseek-coder-1.3b-base," e_count += 1 -",110,1.0 +",110,1.0 2025-09-22T14:00:18.063735,[1],"//Implement a function that takes an non-negative integer and returns an array of the first n // integers that are prime numbers and less than n. // for example: @@ -43331,7 +43331,7 @@ function count_up_to(n){ } ",170.19319999963045,,Completions generated successfully. Ready for display in your IDE.,db8e5bbe-8600-4179-8394-530c387504e3,1,deepseek-ai/deepseek-coder-1.3b-base," if ((i % j === 0)){ -",138,0.9814 +",138,0.9814 2025-09-22T14:00:18.504801,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43375,7 +43375,7 @@ class Problem { } }",441.08490000508027,,Completions generated successfully. Ready for display in your IDE.,b295fbb9-5a8d-4b1b-8ad1-e0f29a9c2f9c,1,deepseek-ai/deepseek-coder-1.3b-base," list.set(list.size() - 1, list.get(list.size() - 1) / operand.get(i + 1)); -",414,1.0 +",414,1.0 2025-09-22T14:00:18.734937,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43414,7 +43414,7 @@ class Problem { } }",229.3569000030402,,Completions generated successfully. Ready for display in your IDE.,41797d63-62a8-4c01-b789-617766960ae3,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList evenIndexed = new ArrayList(); -",203,1.0 +",203,1.0 2025-09-22T14:00:18.871599,[1],"from typing import List def Strongest_Extension(class_name: str, extensions: List[str]) -> str: @@ -43442,7 +43442,7 @@ def Strongest_Extension(class_name: str, extensions: List[str]) -> str: best_extension = extension return class_name + '.' + best_extension",137.0463000057498,,Completions generated successfully. Ready for display in your IDE.,5a09fdf9-be53-45e0-b667-c5a5d642d05b,1,deepseek-ai/deepseek-coder-1.3b-base," for extension in extensions: -",111,1.0 +",111,1.0 2025-09-22T14:00:19.031992,[1],"from typing import List def numerical_letter_grade(grades: List[float]) -> List[str]: @@ -43500,7 +43500,7 @@ def numerical_letter_grade(grades: List[float]) -> List[str]: return final ",159.5690000031027,,Completions generated successfully. Ready for display in your IDE.,46490d0a-1af0-473d-a4d8-60aa5ef33277,1,deepseek-ai/deepseek-coder-1.3b-base," final.append('B+') -",131,1.0 +",131,1.0 2025-09-22T14:00:19.147838,[1],"//Check if two words have the same characters. // >>> same_chars(""eabcdzzzz"", ""dddzzzzzzzddeddabc"") // true @@ -43531,7 +43531,7 @@ function same_chars(s0, s1){ ",116.13710000528954,,Completions generated successfully. Ready for display in your IDE.,8560ae4b-a051-44c8-96c2-ee15923bc1e3,1,deepseek-ai/deepseek-coder-1.3b-base," res = false; -",93,1.0 +",93,1.0 2025-09-22T14:00:19.274430,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43573,7 +43573,7 @@ class Problem { } }",126.72390000079758,,Completions generated successfully. Ready for display in your IDE.,f841f7f4-2bb4-4b9c-99db-4f64d742834a,1,deepseek-ai/deepseek-coder-1.3b-base," strongest = s; -",100,1.0 +",100,1.0 2025-09-22T14:00:19.352879,[1],"//prime_fib returns n-th number that is a Fibonacci number and it's also prime. // >>> prime_fib(1) // 2 @@ -43623,7 +43623,7 @@ function prime_fib(n){ fibNum++; } return fib(fibNum - 1);",,77.84540000284323,,Completions generated successfully. Ready for display in your IDE.,c13d4be5-151a-49cd-b211-d74f8c61209d,1,deepseek-ai/deepseek-coder-1.3b-base," -}",47,1.0 +}",47,1.0 2025-09-22T14:00:19.421906,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -43648,7 +43648,7 @@ function string_xor(a, b){ result = arrayXOR.join(''); return result; }",68.41820000408916,,Completions generated successfully. Ready for display in your IDE.,bd87dc44-9c8e-4ba5-bb20-8d88bcb20191,1,deepseek-ai/deepseek-coder-1.3b-base," - ",42,1.0 + ",42,1.0 2025-09-22T14:00:19.472649,[1],"//Given a positive integer n, return the product of the odd digits. // Return 0 if all digits are even. // For example: @@ -43680,7 +43680,7 @@ function digits(n){ } ",50.77299999538809,,Completions generated successfully. Ready for display in your IDE.,075675dd-9b36-4be0-bb70-2267aa6fffb3,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T14:00:19.778067,[1],"def simplify(x: str, n: str) -> bool: """"""Your task is to implement a function that will simplify the expression x * n. The function returns True if x * n evaluates to a whole number and False @@ -43700,7 +43700,7 @@ function digits(n){ ",305.5038999955286,,Completions generated successfully. Ready for display in your IDE.,6727bc54-7a91-4b93-9814-2b3244dd2bfb,1,deepseek-ai/deepseek-coder-1.3b-base," n_num, n_denom = [int(i) for i in n.split('/')] -",283,1.0 +",283,1.0 2025-09-22T14:00:20.475263,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43726,7 +43726,7 @@ class Problem { int count = 0; for (float i : lst) { if ((i % 2 != 0) && (i > 0 || i < 0)) { - float square = Float.valueOf("""" + Math",671,0.9644 + float square = Float.valueOf("""" + Math",671,0.9644 2025-09-22T14:00:20.737931,[1],"//You will be given the name of a class (a string) and an array of extensions. // The extensions are to be used to load additional classes to the class. The // strength of the extension is as follows: Let CAP be the number of the uppercase @@ -43755,7 +43755,7 @@ function Strongest_Extension(class_name, extensions){ } ",262.7569000032963,,Completions generated successfully. Ready for display in your IDE.,e4db67c2-3c1d-4e13-9668-c32ab2abdb15,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=0 ; i>> is_equal_to_sum_even(4) @@ -43824,7 +43824,7 @@ function is_equal_to_sum_even(n){ return false; } }",84.23769999353681,,Completions generated successfully. Ready for display in your IDE.,0f24734c-8dfe-40f0-a337-39089637c733,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T14:00:21.103283,[1],"//You are given a 2 dimensional data, as a nested arrays, // which is similar to matrix, however, unlike matrices, // each row may contain a different number of columns. @@ -43854,7 +43854,7 @@ function get_row(lst, x){ } ",176.20739999983925,,Completions generated successfully. Ready for display in your IDE.,e7bd4c05-8757-4a23-92a4-51eb793cc8c8,1,deepseek-ai/deepseek-coder-1.3b-base," coordinates.push([i, j]); -",141,1.0 +",141,1.0 2025-09-22T14:00:21.207674,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43886,7 +43886,7 @@ class Problem { } }",104.32150000269758,,Completions generated successfully. Ready for display in your IDE.,8d9e7e36-7e9d-41d5-9165-108d33c897f7,1,deepseek-ai/deepseek-coder-1.3b-base," } -",76,1.0 +",76,1.0 2025-09-22T14:00:21.321384,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43919,7 +43919,7 @@ class Problem { }"," } }",113.15930000273511,,Completions generated successfully. Ready for display in your IDE.,2450e5c9-9bf1-4efe-abef-691533a3f761,1,deepseek-ai/deepseek-coder-1.3b-base," return ret; -",89,1.0 +",89,1.0 2025-09-22T14:00:21.409384,[1],"from typing import List def odd_count(lst: List[str]) -> List[str]: @@ -43940,7 +43940,7 @@ def odd_count(lst: List[str]) -> List[str]: if int(lst[i][j]) % 2 != 0: count += 1 output.append(f""the number of odd elements {count}n the str{count}ng {count} of the {count}nput."")",,87.82709999650251,,Completions generated successfully. Ready for display in your IDE.,15eba87c-3a23-4973-af92-1bf2fe2a119f,1,deepseek-ai/deepseek-coder-1.3b-base," - return output",61,1.0 + return output",61,1.0 2025-09-22T14:00:21.758387,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -43985,7 +43985,7 @@ class Problem { } }",348.6722999950871,,Completions generated successfully. Ready for display in your IDE.,0e943648-bbd4-4bb3-a3e1-5e9f657ae3cc,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < Math.floor(q.size() / 2); i++) { -",322,1.0 +",322,1.0 2025-09-22T14:00:21.896387,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -44017,7 +44017,7 @@ function correct_bracketing(brackets){ } }",137.77309999568388,,Completions generated successfully. Ready for display in your IDE.,643b602d-9b09-4193-b10c-2f07d148111e,1,deepseek-ai/deepseek-coder-1.3b-base," opens.push('('); -",115,0.9736 +",115,0.9736 2025-09-22T14:00:22.055691,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -44044,7 +44044,7 @@ function next_smallest(lst){ } return second_smallest; }",158.65409999969415,,Completions generated successfully. Ready for display in your IDE.,6af2a98f-baaa-4e08-9b5c-8689462a167a,1,deepseek-ai/deepseek-coder-1.3b-base," if(!Array.isArray(lst)){ -",134,1.0 +",134,1.0 2025-09-22T14:00:22.379904,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -44121,7 +44121,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i return a elif float(a_tmp) < b: return b",324.1459999990184,,Completions generated successfully. Ready for display in your IDE.,4e9b2c57-e984-4746-86f3-f958633cb065,1,deepseek-ai/deepseek-coder-1.3b-base," if isinstance(a, (float, int)) and isinstance(b, (float, int)): -",296,0.9914 +",296,0.9914 2025-09-22T14:00:22.462492,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44148,7 +44148,7 @@ class Problem { return Problem.fib((n - 1)) + Problem.fib((n - 2)); } }",,82.44500000000698,,Completions generated successfully. Ready for display in your IDE.,ebcd9951-7ed8-4a78-83ec-ea0d0b5710cc,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T14:00:22.662500,[1],"from typing import List def has_close_elements(numbers: List[float], threshold: float) -> bool: @@ -44165,7 +44165,7 @@ def has_close_elements(numbers: List[float], threshold: float) -> bool: return False ",199.70359999570064,,Completions generated successfully. Ready for display in your IDE.,04a2478d-dc03-428f-98cb-319a15216b56,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(numbers)): -",154,0.996 +",154,0.996 2025-09-22T14:00:22.766925,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -44232,7 +44232,7 @@ function numerical_letter_grade(grades){ return grades_array; }",104.4700999991619,,Completions generated successfully. Ready for display in your IDE.,2af01c75-8c57-4fd4-8a3a-cf36bb75a066,1,deepseek-ai/deepseek-coder-1.3b-base," } -",77,1.0 +",77,1.0 2025-09-22T14:00:22.935620,[1],"def is_multiply_prime(a: int) -> bool: """"""Write a function that returns true if the given number is the multiplication of 3 prime numbers and false otherwise. @@ -44258,7 +44258,7 @@ function numerical_letter_grade(grades){ ",168.28869999881135,,Completions generated successfully. Ready for display in your IDE.,58e47908-8e85-4105-af8c-d406f6a7fa0c,1,deepseek-ai/deepseek-coder-1.3b-base," prime_factors.append(n) -",145,1.0 +",145,1.0 2025-09-22T14:00:23.112475,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -44296,7 +44296,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",176.7887999958475,,Completions generated successfully. Ready for display in your IDE.,e54c04be-ac62-4c90-a601-2fd1b8b6c9fc,1,deepseek-ai/deepseek-coder-1.3b-base," if(isPalindrome(i)){ -",151,1.0 +",151,1.0 2025-09-22T14:00:23.210171,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44341,7 +44341,7 @@ class Problem { } }",97.2676000019419,,Completions generated successfully. Ready for display in your IDE.,8fbaeb9f-c9f1-44bd-819e-ed3499a659c3,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T14:00:23.283835,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44358,7 +44358,7 @@ class Problem { public static ArrayList getPositive(ArrayList l) { return (new ArrayList(l.stream().filter(x -> x > 0).collect(Collectors.toList()))); }",,73.12609999644337,,Completions generated successfully. Ready for display in your IDE.,79de464d-fbab-4a63-8db6-1789f6d4d43c,1,deepseek-ai/deepseek-coder-1.3b-base," -}",50,1.0 +}",50,1.0 2025-09-22T14:00:23.327875,[1],"from typing import List def below_zero(operations: List[int]) -> bool: @@ -44377,7 +44377,7 @@ def below_zero(operations: List[int]) -> bool: return True return False -print(below_zero([1, 2, 3]))",,44.296799998846836,,Completions generated successfully. Ready for display in your IDE.,be04dd2e-8675-4b2f-8c8a-206f26c72559,1,deepseek-ai/deepseek-coder-1.3b-base,,22,1.0 +print(below_zero([1, 2, 3]))",,44.296799998846836,,Completions generated successfully. Ready for display in your IDE.,be04dd2e-8675-4b2f-8c8a-206f26c72559,1,deepseek-ai/deepseek-coder-1.3b-base,,22,1.0 2025-09-22T14:00:23.413873,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44400,7 +44400,7 @@ class Problem { } }",85.95689999492606,,Completions generated successfully. Ready for display in your IDE.,07b1aa30-3900-4ffe-bb4e-dde1c9aa8993,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:23.519877,[1],"from typing import List def double_the_difference(lst: List[float]) -> int: @@ -44426,7 +44426,7 @@ def double_the_difference(lst: List[float]) -> int: ",105.24930000246968,,Completions generated successfully. Ready for display in your IDE.,77a70e49-2ed5-4db1-98f5-6bdb0e9da198,1,deepseek-ai/deepseek-coder-1.3b-base," sum = 0 -",74,1.0 +",74,1.0 2025-09-22T14:00:23.757874,[1],"//Return n-th Fibonacci number. // >>> fib(10) // 55 @@ -44441,7 +44441,7 @@ function fib(n){ } ",237.59940000309143,,Completions generated successfully. Ready for display in your IDE.,7d3dc8b6-48b1-402b-a922-2276b04c4dd8,1,deepseek-ai/deepseek-coder-1.3b-base," return fib(n - 1) + fib(n - 2); -",217,0.9898 +",217,0.9898 2025-09-22T14:00:23.907844,[1],"from typing import Tuple def bf(planet1: str, planet2: str) -> Tuple[str, ...]: @@ -44472,7 +44472,7 @@ def bf(planet1: str, planet2: str) -> Tuple[str, ...]: planets = planets[start+1:end] else: planets = planets[end+1:start]",,149.00409999972908,,Completions generated successfully. Ready for display in your IDE.,1497d21a-90e6-4570-96c9-d93dd29760c8,1,deepseek-ai/deepseek-coder-1.3b-base," - return tuple(planets)",121,0.8948 + return tuple(planets)",121,0.8948 2025-09-22T14:00:24.019604,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44550,7 +44550,7 @@ class Problem { } }",112.05980000522686,,Completions generated successfully. Ready for display in your IDE.,a14a1ba5-3646-44ad-aa94-ae8f214bfa2f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",75,1.0 +",75,1.0 2025-09-22T14:00:24.678564,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44603,7 +44603,7 @@ class Problem { } else { if (day < 1 || day > 29) return valid_date; } -",635,0.9866 +",635,0.9866 2025-09-22T14:00:24.944065,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -44644,7 +44644,7 @@ function int_to_mini_roman(number){ }","} ",264.9245000066003,,Completions generated successfully. Ready for display in your IDE.,70f2580e-24ad-4f16-ac94-48b1a02fd56a,1,deepseek-ai/deepseek-coder-1.3b-base," console.log(""The mini version of the given number is "",output,"".""); -",224,0.9035 +",224,0.9035 2025-09-22T14:00:25.026883,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44672,7 +44672,7 @@ class Problem { } return p; }",,83.006300003035,,Completions generated successfully. Ready for display in your IDE.,af0e25e9-8f01-4f18-b619-8e4c264fad35,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T14:00:25.165661,[1],"//You are given an array arr of integers and you need to return // sum of magnitudes of integers multiplied by product of all signs // of each number in the array, represented by 1, -1 or 0. @@ -44705,7 +44705,7 @@ function prod_signs(arr){ } ",138.28230000217445,,Completions generated successfully. Ready for display in your IDE.,49ff50e0-6f09-4e2b-9cc1-1135e4bfe4df,1,deepseek-ai/deepseek-coder-1.3b-base," console.log(""undefined""); -",116,0.9869 +",116,0.9869 2025-09-22T14:00:25.252771,[1],"//You are given a positive integer n. You have to create an integer array a of length n. // For each i (1 ≤ i ≤ n), the value of a[i] = i * i - i + 1. // Return the number of triples (a[i], a[j], a[k]) of a where i < j < k, @@ -44733,7 +44733,7 @@ function get_max_triples(n){ } ",86.8216000017128,,Completions generated successfully. Ready for display in your IDE.,c78b7ff5-543f-44e9-b07f-e35d57cab191,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:25.342139,[1],"//Return true if a given number is prime, and false otherwise. // >>> is_prime(6) // false @@ -44771,7 +44771,7 @@ function is_prime(n){ } ",89.26110000174958,,Completions generated successfully. Ready for display in your IDE.,bd2de108-8840-4be7-900b-3efae35e545d,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:25.433000,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44797,7 +44797,7 @@ class Problem { } return false;",},91.19749999808846,,Completions generated successfully. Ready for display in your IDE.,451fb06a-ba11-42b0-b7dc-da900dd4bd2b,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:25.498532,[1],"//Create a function that returns true if the last character // of a given string is an alphabetical character and is not // a part of a word, and false otherwise. @@ -44817,7 +44817,7 @@ function check_if_last_char_is_a_letter(txt){ return true; } return false;",,65.06429999717511,,Completions generated successfully. Ready for display in your IDE.,1109df48-a5ff-4388-a02d-a7b772119cbc,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:00:25.587955,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44849,7 +44849,7 @@ class Problem { } return sb.reverse().toString();",},88.86019999772543,,Completions generated successfully. Ready for display in your IDE.,21a75232-84e7-410e-a98d-b276fd2c80d5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",61,1.0 +",61,1.0 2025-09-22T14:00:25.744396,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -44889,7 +44889,7 @@ class Problem { });"," } }",156.05150000192225,,Completions generated successfully. Ready for display in your IDE.,962449f9-d0aa-4270-89c9-78724c27d574,1,deepseek-ai/deepseek-coder-1.3b-base," return words.get(0); -",130,1.0 +",130,1.0 2025-09-22T14:00:25.858541,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -44927,7 +44927,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",113.9444000000367,,Completions generated successfully. Ready for display in your IDE.,6fbaab42-346b-4154-b1f4-c4dfcd6d5c44,1,deepseek-ai/deepseek-coder-1.3b-base," odd++; -",89,1.0 +",89,1.0 2025-09-22T14:00:26.001802,[1],"//Given a positive integer n, return the count of the numbers of n-digit // positive integers that start or end with 1. function starts_one_ends(n){ @@ -44938,7 +44938,7 @@ function starts_one_ends(n){ return ones.length; }",142.60530000319704,,Completions generated successfully. Ready for display in your IDE.,1e2fef4c-05c6-476e-b074-9856863edfff,1,deepseek-ai/deepseek-coder-1.3b-base," ones.push(i) -",105,1.0 +",105,1.0 2025-09-22T14:00:26.275870,[1],"//From a supplied array of numbers (of length at least two) select and return two that are the closest to each // other and return them in order (smaller number, larger number). // >>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) @@ -44960,7 +44960,7 @@ function find_closest_elements(numbers){ } ",274.6268999981112,,Completions generated successfully. Ready for display in your IDE.,66bdbcee-1cba-4edc-94a2-35636cad3fcf,1,deepseek-ai/deepseek-coder-1.3b-base," for (let j = i + 1; j < numbers.length; j++) { -",248,0.9963 +",248,0.9963 2025-09-22T14:00:26.523708,[1],"//Given a grid with N rows and N columns (N >= 2) and a positive integer k, // each cell of the grid contains a value. Every integer in the range [1, N * N] // inclusive appears exactly once on the cells of the grid. @@ -45032,7 +45032,7 @@ function minPath(grid, k){ return path; }",247.12219999491936,,Completions generated successfully. Ready for display in your IDE.,fad15c51-fdf9-4a15-bb1c-5e50e3f7c28d,1,deepseek-ai/deepseek-coder-1.3b-base," neighbours.push(grid[x][y - 1]); -",218,1.0 +",218,1.0 2025-09-22T14:00:26.597688,[1],"//Write a function that returns true if the object q will fly, and false otherwise. // The object q will fly if it's balanced (it is a palindromic array) and the sum of its elements is less than or equal the maximum possible weight w. // Example: @@ -45061,7 +45061,7 @@ function will_it_fly(q, w){ } } return balanced && sum <= w;",,74.01219999883324,,Completions generated successfully. Ready for display in your IDE.,5ebb9fc7-f68f-4715-bd51-79ffe3a7d9eb,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:00:26.808849,[1],"//Input are two strings a and b consisting only of 1s and 0s. // Perform binary XOR on these inputs and return result also as a string. // >>> string_xor(""010"", ""110"") @@ -45087,7 +45087,7 @@ function string_xor(a, b){ return result; }",211.03159999620402,,Completions generated successfully. Ready for display in your IDE.,83784419-3500-4aeb-aacc-91667815a7ab,1,deepseek-ai/deepseek-coder-1.3b-base," arrayB.push(parseInt(b[j])); -",185,1.0 +",185,1.0 2025-09-22T14:00:26.858636,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45115,7 +45115,7 @@ class Problem { if (n == 3) return 0; return fib4(n-1) + fib4(n-2) + fib4(n-3) + fib4(n-4); } -}",49.71369999839226,,Completions generated successfully. Ready for display in your IDE.,7dc18ff4-228b-4c6e-b876-5d95523efa0a,1,deepseek-ai/deepseek-coder-1.3b-base,,22,1.0 +}",49.71369999839226,,Completions generated successfully. Ready for display in your IDE.,7dc18ff4-228b-4c6e-b876-5d95523efa0a,1,deepseek-ai/deepseek-coder-1.3b-base,,22,1.0 2025-09-22T14:00:26.979353,[1],"//You are given a positive integer n. You have to create an integer array a of length n. // For each i (1 ≤ i ≤ n), the value of a[i] = i * i - i + 1. // Return the number of triples (a[i], a[j], a[k]) of a where i < j < k, @@ -45143,7 +45143,7 @@ function get_max_triples(n){ }","} ",120.06259999907343,,Completions generated successfully. Ready for display in your IDE.,e1b1c82d-a1c0-4a9f-948f-23de4a0b0772,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",84,1.0 +",84,1.0 2025-09-22T14:00:27.123818,[1],"//Input to this function is a string containing multiple groups of nested parentheses. Your goal is to // separate those group into separate strings and return the array of those. // Separate groups are balanced (each open brace is properly closed) and not nested within each other @@ -45169,7 +45169,7 @@ function separate_paren_groups(paren_string){ return groups; } ",144.43779999419348,,Completions generated successfully. Ready for display in your IDE.,4aa23145-dbb1-4fb4-9dff-b60fa0d52c92,1,deepseek-ai/deepseek-coder-1.3b-base," // Write your code here... -",104,0.926 +",104,0.926 2025-09-22T14:00:27.440703,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45188,7 +45188,7 @@ class Problem { }",316.29189999512164,,Completions generated successfully. Ready for display in your IDE.,887d022d-2fd2-41ac-9ef8-8afd7000f464,1,deepseek-ai/deepseek-coder-1.3b-base," return numbers.stream() .map(x -> x / ((max-min)*(-1))); -",290,0.939 +",290,0.939 2025-09-22T14:00:27.539568,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45220,7 +45220,7 @@ class Problem { } }",98.76320000330452,,Completions generated successfully. Ready for display in your IDE.,dc8f7cf6-0976-461e-9e90-13e6cc1bb09b,1,deepseek-ai/deepseek-coder-1.3b-base," }); -",69,1.0 +",69,1.0 2025-09-22T14:00:28.232922,[1],"//Imagine a road that's a perfectly straight infinitely long line. // n cars are driving left to right; simultaneously, a different set of n cars // are driving right to left. The two sets of cars start out being very far from @@ -45244,7 +45244,7 @@ function car_race_collision(n){ //for loop to fill arrays with random numbers between -5 and +5 inclusive while (!Array.isArray(left_to_right)) { - const randLeftToRight = Math.floor((Math.random() * ((2*6)-3+1)) + ",669,0.8739 + const randLeftToRight = Math.floor((Math.random() * ((2*6)-3+1)) + ",669,0.8739 2025-09-22T14:00:28.372122,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45277,7 +45277,7 @@ class Problem { } }",138.85529999970458,,Completions generated successfully. Ready for display in your IDE.,9fe752e0-c021-42f8-bc5f-020be1af6e7a,1,deepseek-ai/deepseek-coder-1.3b-base," long v = 0l; -",115,0.9907 +",115,0.9907 2025-09-22T14:00:28.631691,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -45321,7 +45321,7 @@ function histogram(test){ } ",259.06369999574963,,Completions generated successfully. Ready for display in your IDE.,d7f28ce7-19d8-4f63-a5d5-915d653088f7,1,deepseek-ai/deepseek-coder-1.3b-base," else if ((count[key]) === max && !result[key] ) { -",232,0.9359 +",232,0.9359 2025-09-22T14:00:28.819533,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -45348,7 +45348,7 @@ function next_smallest(lst){ return second_smallest; }",187.4863000048208,,Completions generated successfully. Ready for display in your IDE.,7d09b0dc-e609-479b-893a-fac4806f980a,1,deepseek-ai/deepseek-coder-1.3b-base," second_smallest=lst[i]; -",166,1.0 +",166,1.0 2025-09-22T14:00:28.940278,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -45404,7 +45404,7 @@ function by_length(arr){ }",120.50830000225687,,Completions generated successfully. Ready for display in your IDE.,e2d1bb59-9b47-4830-9709-65710d9f956f,1,deepseek-ai/deepseek-coder-1.3b-base," } -",89,0.9326 +",89,0.9326 2025-09-22T14:00:29.318778,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45428,7 +45428,7 @@ class Problem { }",378.756499994779,,Completions generated successfully. Ready for display in your IDE.,3e7217d8-4c57-4c9e-b9ea-ea1bb86cf7c9,1,deepseek-ai/deepseek-coder-1.3b-base," while ((a %= b) != 0 && (b %= a) != 0); return a + b; -",341,0.9635 +",341,0.9635 2025-09-22T14:00:29.922606,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -45480,7 +45480,7 @@ function check_dict_case(dict){ //console.log(""key:"" + key); console.log(""value:"" + value); if (!isNaN(Number(value)) || typeof Number === 'function') { -",576,0.9377 +",576,0.9377 2025-09-22T14:00:30.430938,[1],"//Given a string s and a natural number n, you have been tasked to implement // a function that returns an array of all words from string s that contain exactly // n consonants, in order these words appear in the string s. @@ -45509,7 +45509,7 @@ function select_words(s, n){ console.log([...new Set(selected)]); }; -",477,0.9124 +",477,0.9124 2025-09-22T14:00:31.127567,[1],"//triples_sum_to_zero takes an array of integers as an input. // it returns true if there are three distinct elements in the array that // sum to zero, and false otherwise. @@ -45540,7 +45540,7 @@ function triples_sum_to_zero(l){ console.log(""Sum: ""+sum); //console.log('i:'+i,'j:'+j,'k:'+k); if (sum === 0) { -",673,0.9704 +",673,0.9704 2025-09-22T14:00:31.217829,[1],"def is_prime(n: int) -> bool: """"""Return true if a given number is prime, and false otherwise. >>> is_prime(6) @@ -45564,7 +45564,7 @@ function triples_sum_to_zero(l){ if n % i == 0: return False"," ",89.54990000347607,,Completions generated successfully. Ready for display in your IDE.,00c9419f-5fca-498a-9e19-64545e669abe,1,deepseek-ai/deepseek-coder-1.3b-base," - return True",67,1.0 + return True",67,1.0 2025-09-22T14:00:31.313985,[1],"//"" // This function will take an array of integers. For all entries in the array, the function shall square the integer entry if its index is a // multiple of 3 and will cube the integer entry if its index is a multiple of 4 and not a multiple of 3. The function will not @@ -45589,7 +45589,7 @@ function sum_squares(lst){ return sum; }",96.01579999434762,,Completions generated successfully. Ready for display in your IDE.,7335f2c6-b189-404f-afa5-38ded83abd10,1,deepseek-ai/deepseek-coder-1.3b-base," } -",72,1.0 +",72,1.0 2025-09-22T14:00:31.398462,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45609,7 +45609,7 @@ class Problem { } return c.toString();",},84.72120000078576,,Completions generated successfully. Ready for display in your IDE.,b835eec8-6006-46c0-9761-62cd74e3bc91,1,deepseek-ai/deepseek-coder-1.3b-base," } -",59,1.0 +",59,1.0 2025-09-22T14:00:31.524996,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45647,7 +45647,7 @@ class Problem { } }",126.13910000072792,,Completions generated successfully. Ready for display in your IDE.,11dbf374-3250-45ef-93a2-c2b8ba7ba9b1,1,deepseek-ai/deepseek-coder-1.3b-base," return lst; -",99,1.0 +",99,1.0 2025-09-22T14:00:31.640590,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -45680,7 +45680,7 @@ class Problem { ",115.4750999994576,,Completions generated successfully. Ready for display in your IDE.,31e99ef6-9e56-4927-8201-81834f7d9f3f,1,deepseek-ai/deepseek-coder-1.3b-base," i = 0 -",87,1.0 +",87,1.0 2025-09-22T14:00:31.862425,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -45747,7 +45747,7 @@ function numerical_letter_grade(grades){ return grades_array; }",221.85759999410948,,Completions generated successfully. Ready for display in your IDE.,5af5a80d-31da-41c8-9476-efbc968716f2,1,deepseek-ai/deepseek-coder-1.3b-base," else if (grades[i] > 3.0){ -",191,1.0 +",191,1.0 2025-09-22T14:00:32.227427,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45782,7 +45782,7 @@ class Problem { } }",364.230300001509,,Completions generated successfully. Ready for display in your IDE.,f61ecebf-9ff0-45b5-acbf-5ffcf1b0b3c2,1,deepseek-ai/deepseek-coder-1.3b-base," lst.set(i, lst.get(i) * lst.get(i)); -",285,1.0 +",285,1.0 2025-09-22T14:00:32.326370,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45824,7 +45824,7 @@ class Problem { if (c1 == 0 || c2 == 0) return ""Yes""; else return ""No"";",},98.37230000266572,,Completions generated successfully. Ready for display in your IDE.,c4165945-f017-4554-bf5a-8f9a396fe0e6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T14:00:32.395964,[1],"//Create a function which returns the largest index of an element which // is not greater than or equal to the element immediately preceding it. If // no such element exists then return -1. The given array will not contain @@ -45845,7 +45845,7 @@ function can_arrange(arr){ } } return res;",,69.73949999519391,,Completions generated successfully. Ready for display in your IDE.,e7a2d874-6bcb-4394-a8e9-8509f6223118,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T14:00:32.483569,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -45885,7 +45885,7 @@ class Problem { } }",87.10040000005392,,Completions generated successfully. Ready for display in your IDE.,2c230020-0dbf-430f-8bcd-6d90a3763ccc,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:32.575140,[1],"//Given an array of strings, where each string consists of only digits, return an array. // Each element i of the output should be ""the number of odd elements in the // string i of the input."" where all the i's should be replaced by the number @@ -45908,7 +45908,7 @@ function odd_count(lst){ return new_lst; }",91.52089999406599,,Completions generated successfully. Ready for display in your IDE.,38430288-c8a1-4588-8893-313ed92929d6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:32.743028,[1],"def cycpattern_check(a: str, b: str) -> bool: """"""You are given 2 words. You need to return True if the second word or any of its rotations is a substring in the first word >>> cycpattern_check('abcd', 'abd') @@ -45931,7 +45931,7 @@ function odd_count(lst){ return False ",167.70069999620318,,Completions generated successfully. Ready for display in your IDE.,ff71166e-d528-41cc-848f-392919950e8b,1,deepseek-ai/deepseek-coder-1.3b-base," if len(a) == len(b): -",144,0.9307 +",144,0.9307 2025-09-22T14:00:32.859510,[1],"from typing import List def count_nums(arr: List[int]) -> int: @@ -45961,7 +45961,7 @@ def count_nums(arr: List[int]) -> int: return n ",116.15950000123121,,Completions generated successfully. Ready for display in your IDE.,810ecd7c-38e3-4bb1-aa86-f5b883b70eb4,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",73,1.0 +",73,1.0 2025-09-22T14:00:32.921940,[1],"//Given a string s, count the number of uppercase vowels in even indices. // For example: // >>> count_upper(""aBCdEf"") @@ -45979,7 +45979,7 @@ function count_upper(s){ } return result; }",62.22880000132136,,Completions generated successfully. Ready for display in your IDE.,1eb609ed-53f0-43dd-9307-ae77ce392d31,1,deepseek-ai/deepseek-coder-1.3b-base," -",32,1.0 +",32,1.0 2025-09-22T14:00:33.008642,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -46046,7 +46046,7 @@ function do_algebra(operator, operand){ return answer; } ",86.47170000040205,,Completions generated successfully. Ready for display in your IDE.,b84bd900-5a7a-4d55-a593-baf31bb5e90b,1,deepseek-ai/deepseek-coder-1.3b-base," -",41,1.0 +",41,1.0 2025-09-22T14:00:33.177755,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46079,7 +46079,7 @@ class Problem { } }",168.58399999910034,,Completions generated successfully. Ready for display in your IDE.,75596307-40fc-4e13-af42-a3fc67f42dce,1,deepseek-ai/deepseek-coder-1.3b-base," Arrays.sort(chars); -",135,1.0 +",135,1.0 2025-09-22T14:00:33.875732,[1],"//Task // We are given two strings s and c, you have to deleted all the characters in s that are equal to any character in c // then check if the result string is palindrome. @@ -46106,7 +46106,7 @@ function reverse_delete(s, c){ console.log({arr}); function deleteCharacters(charToDelete:string):void{ - while ((index = arr.indexOf(charToDelete)) !== -1 )",673,0.9245 + while ((index = arr.indexOf(charToDelete)) !== -1 )",673,0.9245 2025-09-22T14:00:33.969873,[1],"//Write a function that accepts an array of strings as a parameter, // deletes the strings that have odd lengths from it, // and returns the resulted array with a sorted order, @@ -46150,7 +46150,7 @@ function sorted_list_sum(lst){ var a = [""aa"", ""a"", ""aaa""]; var b = [""ab"", ""a"", ""aaa"", ""cd""];",93.50500000437023,,Completions generated successfully. Ready for display in your IDE.,51e07160-8886-49ef-98d5-6571edbfe326,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:34.095592,[1],"def starts_one_ends(n: int) -> int: """""" Given a positive integer n, return the count of the numbers of n-digit @@ -46162,7 +46162,7 @@ var b = [""ab"", ""a"", ""aaa"", ""cd""];",93.50500000437023,,Completions genera ",125.92379999841796,,Completions generated successfully. Ready for display in your IDE.,58ba46db-0447-4a40-963f-a3e78bef86fb,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1 -",82,1.0 +",82,1.0 2025-09-22T14:00:34.277251,[1],"//You are given a positive integer n. You have to create an integer array a of length n. // For each i (1 ≤ i ≤ n), the value of a[i] = i * i - i + 1. // Return the number of triples (a[i], a[j], a[k]) of a where i < j < k, @@ -46190,7 +46190,7 @@ function get_max_triples(n){ } ",180.95249999896623,,Completions generated successfully. Ready for display in your IDE.,1ee19b15-bc99-4580-b580-9cabb0f5ff08,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T14:00:34.403730,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46215,7 +46215,7 @@ class Problem { return ret; } }",126.77219999750378,,Completions generated successfully. Ready for display in your IDE.,376d1447-e204-49c8-a128-d2bad23b00df,1,deepseek-ai/deepseek-coder-1.3b-base," long ret = 0L; -",102,1.0 +",102,1.0 2025-09-22T14:00:34.561569,[1],"//Filter an input array of strings only for ones that contain given substring // >>> filter_by_substring([], ""a"") // [] @@ -46229,7 +46229,7 @@ function filter_by_substring(strings, substring){ return filtered; }",157.45260000403505,,Completions generated successfully. Ready for display in your IDE.,14ef5d9e-6bd8-4596-8024-0915bef9bddf,1,deepseek-ai/deepseek-coder-1.3b-base," filtered.push(strings[i]); -",135,1.0 +",135,1.0 2025-09-22T14:00:34.645046,[1],"//Return array of all prefixes from shortest to longest of the input string // >>> all_prefixes(""abc"") // [""a"", ""ab"", ""abc""] @@ -46240,7 +46240,7 @@ function all_prefixes(string){ } ",83.52270000614226,,Completions generated successfully. Ready for display in your IDE.,1b62a3b1-8ace-4c97-b9f0-588a0c5dd953,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:34.713379,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46280,7 +46280,7 @@ class Problem { } return arr.size() == 3; }",,67.72890000138432,,Completions generated successfully. Ready for display in your IDE.,187774ef-26a1-4125-bffd-72f2081e0c37,1,deepseek-ai/deepseek-coder-1.3b-base," -}",45,1.0 +}",45,1.0 2025-09-22T14:00:34.784977,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46306,7 +46306,7 @@ class Problem { } return levels; }",,71.76649999746587,,Completions generated successfully. Ready for display in your IDE.,012fa140-8d2f-48c3-a3c0-c0eb0dea62a6,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:00:34.828571,[1],"//Input to this function is a string representing musical notes in a special ASCII format. // Your task is to parse this string and return array of integers corresponding to how many beats does each // not last. @@ -46329,7 +46329,7 @@ function parse_music(music_string){ }); return song; } -",43.47630000120262,,Completions generated successfully. Ready for display in your IDE.,3451a599-1231-438f-b666-67ce7bd4b697,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 +",43.47630000120262,,Completions generated successfully. Ready for display in your IDE.,3451a599-1231-438f-b666-67ce7bd4b697,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T14:00:34.940514,[1],"from typing import List def f(n: int) -> List[int]: @@ -46351,7 +46351,7 @@ def f(n: int) -> List[int]: return a pass",111.51740000059363,,Completions generated successfully. Ready for display in your IDE.,4f159682-ba6d-4969-b0a0-2571aee76719,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",78,1.0 +",78,1.0 2025-09-22T14:00:35.048025,[1],"from typing import Dict def check_dict_case(dict: Dict[str, str]) -> bool: @@ -46378,7 +46378,7 @@ def check_dict_case(dict: Dict[str, str]) -> bool: if any(not x.islower() for x in dict.keys()): if any(not x.isupper() for x in dict.keys()): return False",,106.94769999827258,,Completions generated successfully. Ready for display in your IDE.,ce53c629-a262-4ca5-8551-3ca0a49a4892,1,deepseek-ai/deepseek-coder-1.3b-base," - return True",63,0.989 + return True",63,0.989 2025-09-22T14:00:35.742963,[1],"//You are given a string representing a sentence, // the sentence contains some words separated by a space, // and you have to return a string that contains the words from the original sentence, @@ -46414,7 +46414,7 @@ function words_in_sentence(sentence){ function getPrimes (arrOfNumbers) { return arrOfNumbers.filter((item) =>{ - item > 3 && Number.",669,0.8763 + item > 3 && Number.",669,0.8763 2025-09-22T14:00:35.884942,[1],"from typing import List def separate_paren_groups(paren_string: str) -> List[str]: @@ -46439,7 +46439,7 @@ def separate_paren_groups(paren_string: str) -> List[str]: return groups ",141.24509999965085,,Completions generated successfully. Ready for display in your IDE.,972912ff-552c-4fb3-9a67-81c6dee0b2d3,1,deepseek-ai/deepseek-coder-1.3b-base," if c == '(': -",117,1.0 +",117,1.0 2025-09-22T14:00:36.011941,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46490,7 +46490,7 @@ class Problem { } }",127.00939999922412,,Completions generated successfully. Ready for display in your IDE.,0de6dfb0-61e2-4ecd-9f0b-e73604b138dd,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1; -",100,1.0 +",100,1.0 2025-09-22T14:00:36.274931,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46525,7 +46525,7 @@ class Problem { } }",262.47320000402397,,Completions generated successfully. Ready for display in your IDE.,e8c7e93b-ef9e-436a-ae60-b57f5ae12f86,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < sz; i++) { -",222,1.0 +",222,1.0 2025-09-22T14:00:36.368933,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46541,7 +46541,7 @@ class Problem { TreeSet tree = new TreeSet(l); return new ArrayList(tree);",},93.97679999528918,,Completions generated successfully. Ready for display in your IDE.,aeb4ca78-91fb-4aa0-a25c-f623f82c3aab,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:36.439931,[1],"//From a given array of integers, generate an array of rolling maximum element found until given moment // in the sequence. // >>> rolling_max([1, 2, 3, 2, 3, 4, 2]) @@ -46556,7 +46556,7 @@ function rolling_max(numbers){ max.push(tempMax); } return max;",,71.4631999944686,,Completions generated successfully. Ready for display in your IDE.,58b3e386-4081-4c09-9277-537b0bc3cd05,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:00:36.627929,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -46623,7 +46623,7 @@ function numerical_letter_grade(grades){ return grades_array; }",187.60330000077374,,Completions generated successfully. Ready for display in your IDE.,b82af78c-aa2e-4b77-ad69-b69bbee5a759,1,deepseek-ai/deepseek-coder-1.3b-base," grades_array.push(""D-""); -",160,1.0 +",160,1.0 2025-09-22T14:00:36.720933,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46656,7 +46656,7 @@ class Problem { } return ret;",},92.64729999995325,,Completions generated successfully. Ready for display in your IDE.,f3e6b443-1bcf-456b-bddd-e20af46f4db1,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:36.832930,[1],"def triangle_area(a: int, b: int, c: int) -> float: """""" Given the lengths of the three sides of a triangle. Return the area of @@ -46677,7 +46677,7 @@ class Problem { ",111.18849999911617,,Completions generated successfully. Ready for display in your IDE.,7fba0db8-cb80-4664-ac62-0ab96e41c2d3,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",80,1.0 +",80,1.0 2025-09-22T14:00:36.903928,[1],"def get_max_triples(n: int) -> int: """""" You are given a positive integer n. You have to create an integer array a of length n. @@ -46709,7 +46709,7 @@ class Problem { return count ",70.50370000069961,,Completions generated successfully. Ready for display in your IDE.,d24cdee8-e371-4ea9-a912-cb25ff45d415,1,deepseek-ai/deepseek-coder-1.3b-base," -",37,1.0 +",37,1.0 2025-09-22T14:00:37.051763,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46751,7 +46751,7 @@ class Problem { } }",148.1701999946381,,Completions generated successfully. Ready for display in your IDE.,4a4670e5-950a-4ad1-a0e5-5ae31c69a77d,1,deepseek-ai/deepseek-coder-1.3b-base," result.add(s); -",120,1.0 +",120,1.0 2025-09-22T14:00:37.211812,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46785,7 +46785,7 @@ class Problem { } }",159.36830000282498,,Completions generated successfully. Ready for display in your IDE.,c9ff9976-8690-49a8-b40a-2ddb44f95d9f,1,deepseek-ai/deepseek-coder-1.3b-base," if (c == '<') { -",135,1.0 +",135,1.0 2025-09-22T14:00:37.296760,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46808,7 +46808,7 @@ class Problem { } }",84.90059999894584,,Completions generated successfully. Ready for display in your IDE.,08bb920f-8a0d-4a0e-aefb-fe590d5ae615,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:37.495762,[1],"//Write a function that takes an array of numbers as input and returns // the number of elements in the array that are greater than 10 and both // first and last digits of a number are odd (1, 3, 5, 7, 9). @@ -46829,7 +46829,7 @@ function specialFilter(nums){ } ",198.1524000002537,,Completions generated successfully. Ready for display in your IDE.,c1a92bd5-850d-48b2-bc0b-9755f4e0968e,1,deepseek-ai/deepseek-coder-1.3b-base," const num = Math.abs(nums[i]); -",174,0.9757 +",174,0.9757 2025-09-22T14:00:37.601760,[1],"def vowels_count(s: str) -> int: """"""Write a function vowels_count which takes a string representing a word as input and returns the number of vowels in the string. @@ -46852,7 +46852,7 @@ function specialFilter(nums){ return count ",105.57460000563879,,Completions generated successfully. Ready for display in your IDE.,47685c9f-b20c-4891-b98c-29372d65c707,1,deepseek-ai/deepseek-coder-1.3b-base," i = 0 -",82,1.0 +",82,1.0 2025-09-22T14:00:37.727760,[1],"def digits(n: int) -> int: """"""Given a positive integer n, return the product of the odd digits. Return 0 if all digits are even. @@ -46868,7 +46868,7 @@ function specialFilter(nums){ d, m = divmod(n, 10) return digits(d) * (m if m % 2 == 1 else 1) ",125.85050000052433,,Completions generated successfully. Ready for display in your IDE.,48145bb8-aded-4a31-9bf3-7bd4282fde4c,1,deepseek-ai/deepseek-coder-1.3b-base," if n < 10: -",102,1.0 +",102,1.0 2025-09-22T14:00:37.854762,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46907,7 +46907,7 @@ class Problem { } }",126.42580000101589,,Completions generated successfully. Ready for display in your IDE.,d93fe5d5-d219-463a-92f2-dc1cada56247,1,deepseek-ai/deepseek-coder-1.3b-base," index++; -",94,1.0 +",94,1.0 2025-09-22T14:00:37.948758,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46947,7 +46947,7 @@ class Problem { } }",94.26100000564475,,Completions generated successfully. Ready for display in your IDE.,552dbc6b-5eb8-4eef-80ea-fcf11d7e78a0,1,deepseek-ai/deepseek-coder-1.3b-base," }); -",68,1.0 +",68,1.0 2025-09-22T14:00:38.213638,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -46980,7 +46980,7 @@ class Problem { } }",264.2120000018622,,Completions generated successfully. Ready for display in your IDE.,e00e4000-af95-454e-b314-c39ce0096dd5,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (brackets.charAt(i) == ')') { -",225,0.954 +",225,0.954 2025-09-22T14:00:38.308426,[1],"//Given a positive integer, obtain its roman numeral equivalent as a string, // and return it in lowercase. // Restrictions: 1 <= num <= 1000 @@ -47021,7 +47021,7 @@ function int_to_mini_roman(number){ } ",95.15259999898262,,Completions generated successfully. Ready for display in your IDE.,0b830f3f-741c-4632-b43a-26061a449f35,1,deepseek-ai/deepseek-coder-1.3b-base," } -",70,1.0 +",70,1.0 2025-09-22T14:00:38.422023,[1],"//Create a function that takes a value (string) representing a number // and returns the closest integer to it. If the number is equidistant // from two integers, round it away from zero. @@ -47051,7 +47051,7 @@ function closest_integer(value){ } ",113.19139999977779,,Completions generated successfully. Ready for display in your IDE.,b8b816eb-9882-486e-90cd-6c21bd433de5,1,deepseek-ai/deepseek-coder-1.3b-base," }else{ -",82,1.0 +",82,1.0 2025-09-22T14:00:38.549484,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -47087,7 +47087,7 @@ function closest_integer(value){ ",127.40959999791812,,Completions generated successfully. Ready for display in your IDE.,124740c5-4b13-48ad-8f85-1a533b6ce03e,1,deepseek-ai/deepseek-coder-1.3b-base," return 'No' -",100,1.0 +",100,1.0 2025-09-22T14:00:38.635765,[1],"//We have an array 'arr' of N integers arr[1], arr[2], ..., arr[N].The // numbers in the array will be randomly ordered. Your task is to determine if // it is possible to get an array sorted in non-decreasing order by performing @@ -47123,7 +47123,7 @@ function move_one_ball(arr){ }"," return true; } }",85.96440000110306,,Completions generated successfully. Ready for display in your IDE.,1a07abae-3143-457e-b09e-e98f54696256,1,deepseek-ai/deepseek-coder-1.3b-base,"else{ -",59,1.0 +",59,1.0 2025-09-22T14:00:38.698005,[1],"//Given an array of numbers, return whether or not they are sorted // in ascending order. If array has more than 1 duplicate of the same // number, return false. Assume no negative numbers and only integers. @@ -47164,7 +47164,7 @@ function is_sorted(lst){ return true; } ",62.118700006976724,,Completions generated successfully. Ready for display in your IDE.,47236fc0-c0d7-4da9-95c0-a018990ae555,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T14:00:39.067791,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47197,7 +47197,7 @@ class Problem { }",369.85850000201026,,Completions generated successfully. Ready for display in your IDE.,e6c8cc04-baa6-4208-aa22-a3e18cb6a68e,1,deepseek-ai/deepseek-coder-1.3b-base," long value = hashmap.get(key).longValue() + 1; hashmap.put(key, value); -",336,1.0 +",336,1.0 2025-09-22T14:00:39.314109,[1],"def count_distinct_characters(string: str) -> int: """""" Given a string, find out how many distinct characters (regardless of case) does it consist of >>> count_distinct_characters('xyzXYZ') @@ -47212,7 +47212,7 @@ class Problem { ",245.9409000002779,,Completions generated successfully. Ready for display in your IDE.,4075ae68-d8de-4fad-82a5-bb09220624d9,1,deepseek-ai/deepseek-coder-1.3b-base," if not l.isspace(): s.add(l.lower()) -",225,0.9705 +",225,0.9705 2025-09-22T14:00:39.555574,[1],"from typing import List def sum_squares(lst: List[float]) -> int: @@ -47239,7 +47239,7 @@ def sum_squares(lst: List[float]) -> int: total += i**2 return total",241.06019999453565,,Completions generated successfully. Ready for display in your IDE.,c3da28fa-6994-4f04-9322-980749254ec7,1,deepseek-ai/deepseek-coder-1.3b-base," lst[i] = math.ceil(lst[i]) -",206,1.0 +",206,1.0 2025-09-22T14:00:39.686749,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47270,7 +47270,7 @@ class Problem { } }",131.01879999885568,,Completions generated successfully. Ready for display in your IDE.,f2b2dac3-fc7f-4e70-a865-3313ce15782f,1,deepseek-ai/deepseek-coder-1.3b-base," boolean dec = true; -",104,1.0 +",104,1.0 2025-09-22T14:00:39.777994,[1],"//Given a string s, count the number of uppercase vowels in even indices. // For example: // >>> count_upper(""aBCdEf"") @@ -47289,7 +47289,7 @@ function count_upper(s){ return result; }",90.30300000449643,,Completions generated successfully. Ready for display in your IDE.,12d87598-58a7-49c4-a95b-c4628d05b569,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:39.958781,[1],"def words_in_sentence(sentence: str) -> str: """""" You are given a string representing a sentence, @@ -47323,7 +47323,7 @@ function count_upper(s){ for word in words:"," prime_words.append(word) return ' '.join(prime_words)",181.08989999745972,,Completions generated successfully. Ready for display in your IDE.,951455f2-aff0-4df3-912a-35f49291fcfd,1,deepseek-ai/deepseek-coder-1.3b-base," if is_prime(len(word)): -",156,1.0 +",156,1.0 2025-09-22T14:00:40.161201,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47365,7 +47365,7 @@ class Problem { } }",202.14679999480722,,Completions generated successfully. Ready for display in your IDE.,d9e3b8bd-d48c-4025-98e8-86c64017b96e,1,deepseek-ai/deepseek-coder-1.3b-base," String s4 = s2+s1; -",162,1.0 +",162,1.0 2025-09-22T14:00:40.227529,[1],"//The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fib4(0) -> 0 // fib4(1) -> 0 @@ -47392,7 +47392,7 @@ function fib4(n){ prev[3] = curr; } return prev[3];",,66.11839999823133,,Completions generated successfully. Ready for display in your IDE.,48f0ea37-c830-419f-9d0b-d0b259f40103,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T14:00:40.321624,[1],"//Given an array of numbers, return whether or not they are sorted // in ascending order. If array has more than 1 duplicate of the same // number, return false. Assume no negative numbers and only integers. @@ -47434,7 +47434,7 @@ function is_sorted(lst){ } ",93.78899999865098,,Completions generated successfully. Ready for display in your IDE.,e5903a22-d900-4132-8c74-ffad8a70ec56,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:40.387448,[1],"//Create a function encrypt that takes a string as an argument and // returns a string encrypted with the alphabet being rotated. // The alphabet should be rotated in a manner such that the letters @@ -47457,7 +47457,7 @@ function encrypt(s){ newString += alphabet[(index + shiftBy) % 26]; } return newString;",,65.35119999898598,,Completions generated successfully. Ready for display in your IDE.,011dfd97-db5d-41c1-a8ff-daa292f27c41,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:00:40.602219,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47499,7 +47499,7 @@ class Problem { } }",214.93599999666912,,Completions generated successfully. Ready for display in your IDE.,88148c46-6f14-40a5-9890-80b5ec49dff7,1,deepseek-ai/deepseek-coder-1.3b-base," int diff = Math.abs(cap-sm); -",174,0.9759 +",174,0.9759 2025-09-22T14:00:40.660242,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47530,7 +47530,7 @@ class Problem { return result; } }",57.99709999701008,,Completions generated successfully. Ready for display in your IDE.,b4466039-2a63-4578-9c09-f89c668ee450,1,deepseek-ai/deepseek-coder-1.3b-base," -",33,1.0 +",33,1.0 2025-09-22T14:00:40.747866,[1],"//You'll be given a string of words, and your task is to count the number // of boredoms. A boredom is a sentence that starts with the word ""I"". // Sentences are delimited by '.', '?' or '!'. @@ -47549,7 +47549,7 @@ function is_bored(S){ } ",86.74789999349741,,Completions generated successfully. Ready for display in your IDE.,1b49715e-3d96-45ae-8932-491addb14f98,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:40.853990,[1],"def is_happy(s: str) -> bool: """"""You are given a string s. Your task is to check if the string is happy or not. @@ -47577,7 +47577,7 @@ function is_bored(S){ ",105.8878999974695,,Completions generated successfully. Ready for display in your IDE.,4bd4cc5b-5717-4597-a0ec-670743488922,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",73,1.0 +",73,1.0 2025-09-22T14:00:40.951782,[1],"def is_simple_power(x: int, n: int) -> bool: """"""Your task is to write a function that returns true if a number x is a simple power of n and false in other cases. @@ -47604,7 +47604,7 @@ function is_bored(S){ return False x /= n"," ",98.12160000001313,,Completions generated successfully. Ready for display in your IDE.,5bd96b51-10ef-4985-9830-66372e4d529d,1,deepseek-ai/deepseek-coder-1.3b-base," - return True",63,0.9364 + return True",63,0.9364 2025-09-22T14:00:41.150730,[1],"def get_closest_vowel(word: str) -> str: """"""You are given a word. Your task is to find the closest vowel that stands between two consonants from the right side of the word (case sensitive). @@ -47630,7 +47630,7 @@ function is_bored(S){ return '' ",198.52030000038212,,Completions generated successfully. Ready for display in your IDE.,dc19b8cc-98fa-47d2-8ea5-d42c5cd2a72e,1,deepseek-ai/deepseek-coder-1.3b-base," for i, c in enumerate(word): -",155,0.9588 +",155,0.9588 2025-09-22T14:00:41.399518,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47695,7 +47695,7 @@ class Problem { } }",249.12890000268817,,Completions generated successfully. Ready for display in your IDE.,cb12ef77-4694-448c-8c4e-2cad6031a3ee,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (currentGrade > 0.7f) { -",216,1.0 +",216,1.0 2025-09-22T14:00:41.562517,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47728,7 +47728,7 @@ class Problem { return ret; } }",161.85489999770653,,Completions generated successfully. Ready for display in your IDE.,aa7286a5-7508-45f4-9d0a-bd9029acc134,1,deepseek-ai/deepseek-coder-1.3b-base," var ret = new ArrayList(); -",136,0.9315 +",136,0.9315 2025-09-22T14:00:41.741722,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47759,7 +47759,7 @@ class Problem { return (res_num % res_denom == 0); } }",179.21799999749055,,Completions generated successfully. Ready for display in your IDE.,cfff13f5-bcaf-4ab7-b958-5dcc271f3cb6,1,deepseek-ai/deepseek-coder-1.3b-base," String[] nums = x.split(""/""); -",152,1.0 +",152,1.0 2025-09-22T14:00:41.817199,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -47809,7 +47809,7 @@ class Problem { return ""Yes""; } }",75.18349999736529,,Completions generated successfully. Ready for display in your IDE.,70f6cbe1-3162-4d09-824e-a4ebf3cb020a,1,deepseek-ai/deepseek-coder-1.3b-base," -",39,1.0 +",39,1.0 2025-09-22T14:00:41.955541,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -47847,7 +47847,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",138.46210000338033,,Completions generated successfully. Ready for display in your IDE.,6ba1eaa1-9666-45dc-9d3d-dd88daa45063,1,deepseek-ai/deepseek-coder-1.3b-base," var even = 0; -",110,1.0 +",110,1.0 2025-09-22T14:00:42.095528,[1],"def is_bored(S: str) -> int: """""" You'll be given a string of words, and your task is to count the number @@ -47868,7 +47868,7 @@ function even_odd_palindrome(n){ ",140.11000000027707,,Completions generated successfully. Ready for display in your IDE.,d28aea95-0125-4009-9c3c-95872dd1a39b,1,deepseek-ai/deepseek-coder-1.3b-base," for sentence in sentences: -",98,1.0 +",98,1.0 2025-09-22T14:00:42.213722,[1],"def special_factorial(n: int) -> int: """"""The Brazilian factorial is defined as: brazilian_factorial(n) = n! * (n-1)! * (n-2)! * ... * 1! @@ -47891,7 +47891,7 @@ function even_odd_palindrome(n){ return n * special_factorial(n-1) * factorial(n-1) ",117.95970000093803,,Completions generated successfully. Ready for display in your IDE.,e36291f1-65cb-4977-b36f-75d5cef3cee0,1,deepseek-ai/deepseek-coder-1.3b-base," def factorial(n): -",95,0.9845 +",95,0.9845 2025-09-22T14:00:42.267610,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -47926,7 +47926,7 @@ function max_fill(grid, capacity){ } maxFill += Math.ceil(sum/capacity); } - return maxFill;",,53.69529999734368,,Completions generated successfully. Ready for display in your IDE.,d6600fb2-017e-4cc5-bd84-060bb621d6f3,1,deepseek-ai/deepseek-coder-1.3b-base,,26,0.8386 + return maxFill;",,53.69529999734368,,Completions generated successfully. Ready for display in your IDE.,d6600fb2-017e-4cc5-bd84-060bb621d6f3,1,deepseek-ai/deepseek-coder-1.3b-base,,26,0.8386 2025-09-22T14:00:42.553222,[1],"//Given an array of numbers, return the sum of squares of the numbers // in the array that are odd. Ignore numbers that are negative or not integers. // >>> double_the_difference([1, 3, 2, 0]) @@ -47948,7 +47948,7 @@ function double_the_difference(lst){ ",285.65530000196304,,Completions generated successfully. Ready for display in your IDE.,9d19388a-e789-4600-9989-ddb44536b311,1,deepseek-ai/deepseek-coder-1.3b-base," var num = Math.pow(lst[i], 2); sum += num; -",262,0.9362 +",262,0.9362 2025-09-22T14:00:42.629319,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -48015,7 +48015,7 @@ function do_algebra(operator, operand){ operator.splice(0, 1); } return answer;",,75.34660000237636,,Completions generated successfully. Ready for display in your IDE.,9c831676-8cc1-4a24-81c7-e6ccdd01d26e,1,deepseek-ai/deepseek-coder-1.3b-base," -}",50,0.9321 +}",50,0.9321 2025-09-22T14:00:42.716684,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48062,7 +48062,7 @@ class Problem { } return count; }",,87.65249999851221,,Completions generated successfully. Ready for display in your IDE.,4b927543-768d-4b50-8954-a21a682e7854,1,deepseek-ai/deepseek-coder-1.3b-base," -}",61,1.0 +}",61,1.0 2025-09-22T14:00:42.808018,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -48101,7 +48101,7 @@ function match_parens(lst){ } return 'No'; }",90.80959999846527,,Completions generated successfully. Ready for display in your IDE.,2c82dc16-9c42-435f-b003-adf5f43d1fe5,1,deepseek-ai/deepseek-coder-1.3b-base," - } ",67,0.6931 + } ",67,0.6931 2025-09-22T14:00:42.965122,[1],"from typing import List def incr_list(l: List[int]) -> List[int]: @@ -48115,7 +48115,7 @@ def incr_list(l: List[int]) -> List[int]: for i in l:"," return res ",156.29579999949783,,Completions generated successfully. Ready for display in your IDE.,856defa8-890b-4972-9feb-8eeed8d84866,1,deepseek-ai/deepseek-coder-1.3b-base," res.append(i + 1) -",134,0.9814 +",134,0.9814 2025-09-22T14:00:43.169983,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48177,7 +48177,7 @@ class Problem { } }",204.75080000323942,,Completions generated successfully. Ready for display in your IDE.,28aff54b-8d37-41c3-9f2b-d8cf564803d1,1,deepseek-ai/deepseek-coder-1.3b-base," largestPrime /= 10; -",163,1.0 +",163,1.0 2025-09-22T14:00:43.277547,[1],"//Return 2^n modulo p (be aware of numerics). // >>> modp(3, 5) // 3 @@ -48199,7 +48199,7 @@ function modp(n, p){ ",107.5732999961474,,Completions generated successfully. Ready for display in your IDE.,915a4eab-1019-4c81-a092-d5b509948734,1,deepseek-ai/deepseek-coder-1.3b-base," var result = 1; -",84,1.0 +",84,1.0 2025-09-22T14:00:43.547155,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48241,7 +48241,7 @@ class Problem { } }",269.63119999709306,,Completions generated successfully. Ready for display in your IDE.,dcbf730f-3b28-4639-890c-8b10bc268bda,1,deepseek-ai/deepseek-coder-1.3b-base," sb.append(Character.toUpperCase(s.charAt(i))); -",244,1.0 +",244,1.0 2025-09-22T14:00:43.731131,[1],"def greatest_common_divisor(a: int, b: int) -> int: """""" Return a greatest common divisor of two integers a and b >>> greatest_common_divisor(3, 5) @@ -48255,7 +48255,7 @@ class Problem { ",183.35970000043744,,Completions generated successfully. Ready for display in your IDE.,9065323f-1ad2-4b6c-8632-246e5b8a7fb8,1,deepseek-ai/deepseek-coder-1.3b-base," while (b % a) != 0: -",161,0.9577 +",161,0.9577 2025-09-22T14:00:43.920092,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48300,7 +48300,7 @@ class Problem { } }",188.50120000570314,,Completions generated successfully. Ready for display in your IDE.,925b3e3a-725a-4f6a-aaae-dd263f3fbe66,1,deepseek-ai/deepseek-coder-1.3b-base," if (word.length() == prime) { -",161,1.0 +",161,1.0 2025-09-22T14:00:44.013106,[1],"//Given a positive integer n, return an array that has the number of even and odd // integer palindromes that fall within the range(1, n), inclusive. // Example 1: @@ -48338,7 +48338,7 @@ function even_odd_palindrome(n){ return [even, odd]; }",93.56629999820143,,Completions generated successfully. Ready for display in your IDE.,33942b97-f24b-48a1-b3d2-342ef8933d02,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:44.150091,[1],"//Given a positive integer n, return a sorted array that has the odd numbers in collatz sequence. // The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined // as follows: start with any positive integer n. Then each term is obtained from the @@ -48381,7 +48381,7 @@ function get_odd_collatz(n){ } } ",136.40469999518245,,Completions generated successfully. Ready for display in your IDE.,441c11f3-a3f1-4a9e-906b-1abc24b3edc6,1,deepseek-ai/deepseek-coder-1.3b-base," // Write your code here... -",111,0.9215 +",111,0.9215 2025-09-22T14:00:44.338095,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48459,7 +48459,7 @@ class Problem { } }",187.21029999869643,,Completions generated successfully. Ready for display in your IDE.,5034ccb1-8efa-4945-956a-5d3f074cea2f,1,deepseek-ai/deepseek-coder-1.3b-base," while (num >= 100) { -",156,1.0 +",156,1.0 2025-09-22T14:00:45.046092,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48500,7 +48500,7 @@ class Problem { if(!result.isEmpty()) { long val = result.remove(result.size()-1); System.out.println(""val "" + val); - for (;",678,0.945 + for (;",678,0.945 2025-09-22T14:00:45.214097,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -48555,7 +48555,7 @@ function by_length(arr){ return arr3; }",167.05799999908777,,Completions generated successfully. Ready for display in your IDE.,f9fc0c92-3526-44b6-89ee-ff9680d16d71,1,deepseek-ai/deepseek-coder-1.3b-base," arr3.push(""Nine""); -",140,1.0 +",140,1.0 2025-09-22T14:00:45.310090,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48587,7 +48587,7 @@ class Problem { } }",95.87879999889992,,Completions generated successfully. Ready for display in your IDE.,46e53292-7256-46b4-bf1a-93cec1350a59,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:45.438091,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48627,7 +48627,7 @@ class Problem { } }",127.85339999390999,,Completions generated successfully. Ready for display in your IDE.,232c1b76-1cb9-4082-8c94-b0dfc18a201e,1,deepseek-ai/deepseek-coder-1.3b-base," unique = false; -",98,1.0 +",98,1.0 2025-09-22T14:00:45.522090,[1],"from typing import List def sum_squares(lst: List[float]) -> int: @@ -48654,7 +48654,7 @@ def sum_squares(lst: List[float]) -> int: for i in lst: total += i**2 return total",83.70209999702638,,Completions generated successfully. Ready for display in your IDE.,bf17f12f-7088-4d24-bc75-62953a3adb9b,1,deepseek-ai/deepseek-coder-1.3b-base," import math -",62,0.9378 +",62,0.9378 2025-09-22T14:00:45.779548,[1],"//For a given string, flip lowercase characters to uppercase and uppercase to lowercase. // >>> flip_case(""Hello"") // ""hELLO"" @@ -48669,7 +48669,7 @@ function flip_case(string){ } ",256.7272999949637,,Completions generated successfully. Ready for display in your IDE.,a6e3beba-7510-4296-8e2a-1a788c3f9796,1,deepseek-ai/deepseek-coder-1.3b-base," if (string[i] == string[i].toUpperCase()){ -",227,0.9369 +",227,0.9369 2025-09-22T14:00:45.863216,[1],"//Create a function which takes a string representing a file's name, and returns // 'Yes' if the the file's name is valid, and returns 'No' otherwise. // A file's name is considered to be valid if and only if all the following conditions @@ -48696,7 +48696,7 @@ function file_name_check(file_name){ ",83.58710000175051,,Completions generated successfully. Ready for display in your IDE.,b8cc9e21-4c38-4815-90d6-6a777596ebbe,1,deepseek-ai/deepseek-coder-1.3b-base," -",58,1.0 +",58,1.0 2025-09-22T14:00:45.958693,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48736,7 +48736,7 @@ class Problem { } }",95.6951999978628,,Completions generated successfully. Ready for display in your IDE.,45d9e4b2-f242-4893-9ac0-3e56cb8b99e2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:46.061284,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -48785,7 +48785,7 @@ function check_dict_case(dict){ return true; }",102.36969999823486,,Completions generated successfully. Ready for display in your IDE.,6474c302-699b-4960-83fc-ed44b01d86fb,1,deepseek-ai/deepseek-coder-1.3b-base," } -",66,1.0 +",66,1.0 2025-09-22T14:00:46.169350,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48815,7 +48815,7 @@ class Problem { factors.add(n);"," } }",107.25719999754801,,Completions generated successfully. Ready for display in your IDE.,d907c9d9-e8c2-4aca-8ce6-02c4bb17756a,1,deepseek-ai/deepseek-coder-1.3b-base," return factors; -",83,1.0 +",83,1.0 2025-09-22T14:00:46.336490,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48841,7 +48841,7 @@ class Problem { } }",167.15710000426043,,Completions generated successfully. Ready for display in your IDE.,f62f098d-8cec-437b-9ad3-f308c2a56f72,1,deepseek-ai/deepseek-coder-1.3b-base," sum += lst.get(i); -",144,1.0 +",144,1.0 2025-09-22T14:00:46.412889,[1],"//brackets is a string of ""<"" and "">"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""<"") @@ -48867,7 +48867,7 @@ function correct_bracketing(brackets){ } ",76.31169999513077,,Completions generated successfully. Ready for display in your IDE.,c8b1575b-b4e4-48a5-886a-bbfd7ddc9a47,1,deepseek-ai/deepseek-coder-1.3b-base," } -",55,1.0 +",55,1.0 2025-09-22T14:00:46.535218,[1],"from typing import List def exchange(lst1: List[int], lst2: List[int]) -> str: @@ -48895,7 +48895,7 @@ def exchange(lst1: List[int], lst2: List[int]) -> str: else: return ""YES""",122.27100000018254,,Completions generated successfully. Ready for display in your IDE.,d1330a03-9eb2-456e-b793-efe097f14e73,1,deepseek-ai/deepseek-coder-1.3b-base," if count > 0: -",99,0.9956 +",99,0.9956 2025-09-22T14:00:46.623383,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -48924,7 +48924,7 @@ class Problem { }); return String.join("" "", split);",},88.00029999838443,,Completions generated successfully. Ready for display in your IDE.,9325f95a-1127-469a-9120-acf465cd698e,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:46.700432,[1],"//Return array of prime factors of given integer in the order from smallest to largest. // Each of the factors should be arrayed number of times corresponding to how many times it appeares in factorization. // Input number should be equal to the product of all factors @@ -48947,7 +48947,7 @@ function factorize(n){ } ",76.7615000004298,,Completions generated successfully. Ready for display in your IDE.,9c7a0bc4-aeb4-4d53-b65f-1340ffd623d5,1,deepseek-ai/deepseek-coder-1.3b-base," } -",55,1.0 +",55,1.0 2025-09-22T14:00:46.842162,[1],"//This function takes an array l and returns an array l' such that // l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal // to the values of the corresponding indicies of l, but sorted. @@ -48977,7 +48977,7 @@ function sort_third(l){ return x; }",141.6962000002968,,Completions generated successfully. Ready for display in your IDE.,ea5fce9f-9e89-4292-8215-9fe35944971d,1,deepseek-ai/deepseek-coder-1.3b-base," k=k+1; -",110,0.9651 +",110,0.9651 2025-09-22T14:00:46.986242,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49015,7 +49015,7 @@ class Problem { } }",143.7277000004542,,Completions generated successfully. Ready for display in your IDE.,65f61724-48bb-4e89-8334-ec0487ada77c,1,deepseek-ai/deepseek-coder-1.3b-base," return lst1; -",117,1.0 +",117,1.0 2025-09-22T14:00:47.029848,[1],"def vowels_count(s: str) -> int: """"""Write a function vowels_count which takes a string representing a word as input and returns the number of vowels in the string. @@ -49037,7 +49037,7 @@ class Problem { count += 1 i += 1 return count -",43.295899995428044,,Completions generated successfully. Ready for display in your IDE.,87224c11-f1b5-4d99-b628-0ee9f997920b,1,deepseek-ai/deepseek-coder-1.3b-base,,15,1.0 +",43.295899995428044,,Completions generated successfully. Ready for display in your IDE.,87224c11-f1b5-4d99-b628-0ee9f997920b,1,deepseek-ai/deepseek-coder-1.3b-base,,15,1.0 2025-09-22T14:00:47.094245,[1],"//Concatenate array of strings into a single string // >>> concatenate([]) // """" @@ -49050,7 +49050,7 @@ function concatenate(strings){ } ",64.16429999808315,,Completions generated successfully. Ready for display in your IDE.,96bc36ca-35df-452b-8e9e-cec816354ff4,1,deepseek-ai/deepseek-coder-1.3b-base," } -",43,0.9577 +",43,0.9577 2025-09-22T14:00:47.232561,[1],"from typing import List def rescale_to_unit(numbers: List[float]) -> List[float]: @@ -49063,7 +49063,7 @@ def rescale_to_unit(numbers: List[float]) -> List[float]: return [(val - _min) / (_max - _min) for val in numbers] ",137.70569999906002,,Completions generated successfully. Ready for display in your IDE.,a91b9805-0d45-4ec8-b10c-a7b7da4d175a,1,deepseek-ai/deepseek-coder-1.3b-base," if _min == _max: -",117,0.9537 +",117,0.9537 2025-09-22T14:00:47.431869,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49101,7 +49101,7 @@ class Problem { } }",198.91260000440525,,Completions generated successfully. Ready for display in your IDE.,821e2f8b-cb51-45d6-89fe-61f479b7e0a8,1,deepseek-ai/deepseek-coder-1.3b-base," for (String s : lst1) { -",169,1.0 +",169,1.0 2025-09-22T14:00:47.482590,[1],"from typing import List def count_up_to(n: int) -> List[int]: @@ -49132,7 +49132,7 @@ def count_up_to(n: int) -> List[int]: result.append(number) return result ",50.84749999514315,,Completions generated successfully. Ready for display in your IDE.,46743131-013f-420d-8708-7c115110d127,1,deepseek-ai/deepseek-coder-1.3b-base," -",27,1.0 +",27,1.0 2025-09-22T14:00:47.538321,[1],"//Return array of prime factors of given integer in the order from smallest to largest. // Each of the factors should be arrayed number of times corresponding to how many times it appeares in factorization. // Input number should be equal to the product of all factors @@ -49154,7 +49154,7 @@ function factorize(n){ } } return arr;",,55.37499999627471,,Completions generated successfully. Ready for display in your IDE.,878f856a-0156-4e5a-a20e-c6202b0b7e7d,1,deepseek-ai/deepseek-coder-1.3b-base," -}",35,0.9784 +}",35,0.9784 2025-09-22T14:00:47.663318,[1],"from typing import List def minPath(grid: List[List[int]], k: int) -> List[int]: @@ -49202,7 +49202,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: path.append(min_val) return path",125.0235999978031,,Completions generated successfully. Ready for display in your IDE.,5969c117-2e25-4cba-b830-c068c584578c,1,deepseek-ai/deepseek-coder-1.3b-base," row = i -",97,1.0 +",97,1.0 2025-09-22T14:00:47.750844,[1],"def multiply(a: int, b: int) -> int: """"""Complete the function that takes two integers and returns the product of their unit digits. @@ -49218,7 +49218,7 @@ def minPath(grid: List[List[int]], k: int) -> List[int]: 20 """""" "," -",87.63560000079451,,Completions generated successfully. Ready for display in your IDE.,b6dae7c7-6c6b-404a-b7da-29626fc98fc6,1,deepseek-ai/deepseek-coder-1.3b-base, return a * b,66,1.0 +",87.63560000079451,,Completions generated successfully. Ready for display in your IDE.,b6dae7c7-6c6b-404a-b7da-29626fc98fc6,1,deepseek-ai/deepseek-coder-1.3b-base, return a * b,66,1.0 2025-09-22T14:00:47.850632,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49267,7 +49267,7 @@ class Problem { } }",99.25929999735672,,Completions generated successfully. Ready for display in your IDE.,3717da84-8b9a-444b-bfb6-d80bdf62659a,1,deepseek-ai/deepseek-coder-1.3b-base," } -",71,1.0 +",71,1.0 2025-09-22T14:00:48.335832,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49315,7 +49315,7 @@ class Problem { } } }",484.610799998336,,Completions generated successfully. Ready for display in your IDE.,4c01768d-9fce-432f-a610-fd38de4943b8,1,deepseek-ai/deepseek-coder-1.3b-base," List planets = Arrays.asList(""Mercury"",""Venus"",""Earth"",""Mars"",""Jupiter"",""Saturn"",""Uranus"",""Neptune""); -",456,1.0 +",456,1.0 2025-09-22T14:00:48.410707,[1],"//The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: // fib4(0) -> 0 // fib4(1) -> 0 @@ -49342,7 +49342,7 @@ function fib4(n){ return prev[3]; } ",74.42969999829074,,Completions generated successfully. Ready for display in your IDE.,0e5bdb7b-02be-4d97-abec-761a9c0300db,1,deepseek-ai/deepseek-coder-1.3b-base,"else{ -",51,0.981 +",51,0.981 2025-09-22T14:00:48.474462,[1],"//Imagine a road that's a perfectly straight infinitely long line. // n cars are driving left to right; simultaneously, a different set of n cars // are driving right to left. The two sets of cars start out being very far from @@ -49363,7 +49363,7 @@ function car_race_collision(n){ } } return num_collisions;",,64.21659999614349,,Completions generated successfully. Ready for display in your IDE.,b3e8fb2b-61b1-4fdd-8b1c-672bf26998e4,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,0.9618 +}",41,0.9618 2025-09-22T14:00:48.728620,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49393,7 +49393,7 @@ class Problem { } }",253.37519999447977,,Completions generated successfully. Ready for display in your IDE.,411d3c48-ee77-42e9-9117-39654b08019b,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < text.length(); i++) { -",229,0.9964 +",229,0.9964 2025-09-22T14:00:48.837571,[1],"//Create a function that takes integers, floats, or strings representing // real numbers, and returns the larger variable in its given variable type. // Return undefined if the values are equal. @@ -49435,7 +49435,7 @@ function compare_one(a, b){ } ",109.03069999767467,,Completions generated successfully. Ready for display in your IDE.,1a76a083-94fb-439e-8375-083687c7b5b2,1,deepseek-ai/deepseek-coder-1.3b-base," return undefined; -",84,1.0 +",84,1.0 2025-09-22T14:00:48.977663,[1],"//Input is a space-delimited string of numberals from 'zero' to 'nine'. // Valid choices are 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight' and 'nine'. // Return the string with numbers sorted from smallest to largest @@ -49460,7 +49460,7 @@ function sort_numbers(numbers){ return numString; }",139.71190000302158,,Completions generated successfully. Ready for display in your IDE.,1e4ab56c-395a-4912-bfbe-47c9f05ef0e9,1,deepseek-ai/deepseek-coder-1.3b-base," ""nine"": 9 -",115,0.9916 +",115,0.9916 2025-09-22T14:00:49.157296,[1],"from typing import List def unique_digits(x: List[int]) -> List[int]: @@ -49485,7 +49485,7 @@ def unique_digits(x: List[int]) -> List[int]: temp=temp//10 for i in s: x.remove(i)",,179.9278999969829,,Completions generated successfully. Ready for display in your IDE.,f09f436c-27d6-47ed-9f41-a6f518d4dc0e,1,deepseek-ai/deepseek-coder-1.3b-base," - return sorted(list(set(x)))",145,0.9421 + return sorted(list(set(x)))",145,0.9421 2025-09-22T14:00:49.353736,[1],"//You are given a word. Your task is to find the closest vowel that stands between // two consonants from the right side of the word (case sensitive). // Vowels in the beginning and ending doesn't count. Return empty string if you didn't @@ -49525,7 +49525,7 @@ function get_closest_vowel(word){ break; }else{ continue; -",170,0.9726 +",170,0.9726 2025-09-22T14:00:49.466731,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49603,7 +49603,7 @@ class Problem { } }",112.08469999837689,,Completions generated successfully. Ready for display in your IDE.,8af8c720-5f76-461f-ba33-206dc0040d49,1,deepseek-ai/deepseek-coder-1.3b-base," } -",78,1.0 +",78,1.0 2025-09-22T14:00:49.552740,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49632,7 +49632,7 @@ class Problem { } }",85.99669999966864,,Completions generated successfully. Ready for display in your IDE.,1fcf8381-4acb-469d-b132-884544fd4892,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:49.686247,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49694,7 +49694,7 @@ class Problem { } }",133.02970000222558,,Completions generated successfully. Ready for display in your IDE.,b953fcf4-33fe-4622-afd9-82d651c69ab8,1,deepseek-ai/deepseek-coder-1.3b-base," else { -",101,0.9689 +",101,0.9689 2025-09-22T14:00:49.783663,[1],"//Given an array of integers, sort the integers that are between 1 and 9 inclusive, // reverse the resulting array, and then replace each digit by its corresponding name from // ""One"", ""Two"", ""Three"", ""Four"", ""Five"", ""Six"", ""Seven"", ""Eight"", ""Nine"". @@ -49749,7 +49749,7 @@ function by_length(arr){ return arr3; }",97.28499999619089,,Completions generated successfully. Ready for display in your IDE.,ecdcf00b-0074-4809-9cee-15d12d4394d3,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:49.987684,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49814,7 +49814,7 @@ class Problem { } }",204.01649999985239,,Completions generated successfully. Ready for display in your IDE.,0cacfb7f-eadc-449d-8b8f-d26a809de169,1,deepseek-ai/deepseek-coder-1.3b-base," letterGrades.add(""B+""); -",165,1.0 +",165,1.0 2025-09-22T14:00:50.158534,[1],"def cycpattern_check(a: str, b: str) -> bool: """"""You are given 2 words. You need to return True if the second word or any of its rotations is a substring in the first word >>> cycpattern_check('abcd', 'abd') @@ -49837,7 +49837,7 @@ class Problem { ",170.19230000005336,,Completions generated successfully. Ready for display in your IDE.,8e2ea3a8-f0df-466a-a57b-afceb2326865,1,deepseek-ai/deepseek-coder-1.3b-base," if (b == a[-i:]): -",138,0.9309 +",138,0.9309 2025-09-22T14:00:50.367120,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49867,7 +49867,7 @@ class Problem { } }",208.80619999661576,,Completions generated successfully. Ready for display in your IDE.,334537dc-1e03-4030-9796-858ee18f4856,1,deepseek-ai/deepseek-coder-1.3b-base," newString.append(reverse.substring(1)); -",185,0.9971 +",185,0.9971 2025-09-22T14:00:50.466989,[1],"//You are given 2 words. You need to return true if the second word or any of its rotations is a substring in the first word // >>> cycpattern_check(""abcd"", ""abd"") // false @@ -49889,7 +49889,7 @@ function cycpattern_check(a, b){ } ",99.34380000049714,,Completions generated successfully. Ready for display in your IDE.,7fa41cb8-0233-4c1c-9fd7-0bd3c92d02e2,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:00:50.775748,[1],"def starts_one_ends(n: int) -> int: """""" Given a positive integer n, return the count of the numbers of n-digit @@ -49901,7 +49901,7 @@ function cycpattern_check(a, b){ ",308.73260000225855,,Completions generated successfully. Ready for display in your IDE.,05f2e04c-a644-47a4-99d7-881f9ae25280,1,deepseek-ai/deepseek-coder-1.3b-base," if str(i)[0] == '1' or str(i)[-1] == '1': -",284,0.9815 +",284,0.9815 2025-09-22T14:00:50.936660,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49939,7 +49939,7 @@ class Problem { } }",161.04460000497056,,Completions generated successfully. Ready for display in your IDE.,6e3af699-4eb6-474d-8fd2-21abf2ffc9d0,1,deepseek-ai/deepseek-coder-1.3b-base," if (min < max) { -",134,1.0 +",134,1.0 2025-09-22T14:00:51.071570,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -49986,7 +49986,7 @@ class Problem { }"," } }",133.9732999986154,,Completions generated successfully. Ready for display in your IDE.,592c39a2-c28e-4e81-95d4-f9d600a86614,1,deepseek-ai/deepseek-coder-1.3b-base," return count; -",104,1.0 +",104,1.0 2025-09-22T14:00:51.167687,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50006,7 +50006,7 @@ class Problem { } }",95.83159999601776,,Completions generated successfully. Ready for display in your IDE.,3ef02a04-c634-410c-9856-e33611c94feb,1,deepseek-ai/deepseek-coder-1.3b-base," } -",72,1.0 +",72,1.0 2025-09-22T14:00:51.293705,[1],"def file_name_check(file_name: str) -> str: """"""Create a function which takes a string representing a file's name, and returns 'Yes' if the the file's name is valid, and returns 'No' otherwise. @@ -50042,7 +50042,7 @@ class Problem { ",126.43329999991693,,Completions generated successfully. Ready for display in your IDE.,fc527aa8-d1e3-44e3-b7ed-69636909ba8f,1,deepseek-ai/deepseek-coder-1.3b-base," return 'No' -",100,1.0 +",100,1.0 2025-09-22T14:00:51.438905,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50057,7 +50057,7 @@ class Problem { public static float triangleArea(long a, long h) { "," } }",144.95309999620076,,Completions generated successfully. Ready for display in your IDE.,116e7d95-974d-46af-802d-3177a01fcdd0,1,deepseek-ai/deepseek-coder-1.3b-base," return ((a * h)/2); -",123,0.9755 +",123,0.9755 2025-09-22T14:00:52.134294,[1],"//You are given a rectangular grid of wells. Each row represents a single well, // and each 1 in a row represents a single unit of water. // Each well has a corresponding bucket that can be used to extract water from it, @@ -50100,7 +50100,7 @@ function max_fill(grid, capacity){ const c = grid[0].length; if (r === 0 || !c ){return ""Invalid Input""} - function",670,0.8946 + function",670,0.8946 2025-09-22T14:00:52.189290,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50132,7 +50132,7 @@ class Problem { return ret; } }",54.82740000297781,,Completions generated successfully. Ready for display in your IDE.,4e6e9c6a-1adc-4c90-b76c-27ce0acb25de,1,deepseek-ai/deepseek-coder-1.3b-base," -",33,1.0 +",33,1.0 2025-09-22T14:00:52.249663,[1],"//There are eight planets in our solar system: the closerst to the Sun // is Mercury, the next one is Venus, then Earth, Mars, Jupiter, Saturn, // Uranus, Neptune. @@ -50175,7 +50175,7 @@ function bf(planet1, planet2){ } return res; }",60.55510000442155,,Completions generated successfully. Ready for display in your IDE.,331ef0e8-b0d6-49be-bc5c-92661b4a75a6,1,deepseek-ai/deepseek-coder-1.3b-base," -",36,1.0 +",36,1.0 2025-09-22T14:00:52.527812,[1],"//From a supplied array of numbers (of length at least two) select and return two that are the closest to each // other and return them in order (smaller number, larger number). // >>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) @@ -50197,7 +50197,7 @@ function find_closest_elements(numbers){ } ",277.73570000135805,,Completions generated successfully. Ready for display in your IDE.,b7ba423a-dfbf-4855-beec-dec4db7a1445,1,deepseek-ai/deepseek-coder-1.3b-base," if ((Math.abs(numbers[i] - numbers[j])) <= closest ){ -",255,0.9968 +",255,0.9968 2025-09-22T14:00:52.675683,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50221,7 +50221,7 @@ class Problem { } }",147.84769999823766,,Completions generated successfully. Ready for display in your IDE.,fc624532-fc3b-4329-a2b8-d04b80fb4987,1,deepseek-ai/deepseek-coder-1.3b-base," result.add(delimeter); -",124,1.0 +",124,1.0 2025-09-22T14:00:52.903158,[1],"//Find the shortest palindrome that begins with a supplied string. // Algorithm idea is simple: // - Find the longest postfix of supplied string that is a palindrome. @@ -50246,7 +50246,7 @@ function make_palindrome(string){ } ",226.88170000037644,,Completions generated successfully. Ready for display in your IDE.,45359e0e-81f0-4925-8d4f-58841ff76cc8,1,deepseek-ai/deepseek-coder-1.3b-base," if (is_palindrome(string+=""_"")){ -",203,0.966 +",203,0.966 2025-09-22T14:00:52.999811,[1],"//Given a string of words, return an array of words split on whitespace, if no whitespaces exists in the text you // should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the // alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 @@ -50275,7 +50275,7 @@ function split_words(txt){ } }",96.77969999756897,,Completions generated successfully. Ready for display in your IDE.,fb049a75-903b-4910-bf78-dd70dc6641d7,1,deepseek-ai/deepseek-coder-1.3b-base," else{ -",72,1.0 +",72,1.0 2025-09-22T14:00:53.703251,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50311,7 +50311,7 @@ class Problem { switch (digit) { case 1:c='1';break; case 2:c='2';break; - case 3:c='3",670,0.99 + case 3:c='3",670,0.99 2025-09-22T14:00:53.788245,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50343,7 +50343,7 @@ class Problem { } return out; }",,84.6873000045889,,Completions generated successfully. Ready for display in your IDE.,a67edd49-fb7c-45bb-a92f-95483d28e303,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T14:00:53.872081,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50386,7 +50386,7 @@ class Problem { } return false;",},83.63860000099521,,Completions generated successfully. Ready for display in your IDE.,9e7e5115-08d2-48a3-8352-33849fcbb5ca,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:53.925124,[1],"//You are given an array arr of integers and you need to return // sum of magnitudes of integers multiplied by product of all signs // of each number in the array, represented by 1, -1 or 0. @@ -50418,7 +50418,7 @@ function prod_signs(arr){ } } ",53.12989999947604,,Completions generated successfully. Ready for display in your IDE.,62ddc260-4555-4867-a192-8e5355295263,1,deepseek-ai/deepseek-coder-1.3b-base," -",31,1.0 +",31,1.0 2025-09-22T14:00:54.032171,[1],"//Given the lengths of the three sides of a triangle. Return the area of // the triangle rounded to 2 decimal points if the three sides form a valid triangle. // Otherwise return -1 @@ -50437,7 +50437,7 @@ function triangle_area(a, b, c){ } ",106.77830000349786,,Completions generated successfully. Ready for display in your IDE.,2110ba2a-853a-4528-98e0-19e6bce5580b,1,deepseek-ai/deepseek-coder-1.3b-base," }else{ -",82,0.9892 +",82,0.9892 2025-09-22T14:00:54.280730,[1],"//Write a function count_nums which takes an array of integers and returns // the number of elements which has a sum of digits > 0. // If a number is negative, then its first signed digit will be negative: @@ -50471,7 +50471,7 @@ function count_nums(arr){ return count; }",248.0954000056954,,Completions generated successfully. Ready for display in your IDE.,6401d250-9aa8-41d5-879f-0044b3b195d6,1,deepseek-ai/deepseek-coder-1.3b-base," for(var j = 2; j < str.length; j++) { -",225,1.0 +",225,1.0 2025-09-22T14:00:54.519808,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50519,7 +50519,7 @@ class Problem { } }",238.67219999374356,,Completions generated successfully. Ready for display in your IDE.,2854e2e3-a5af-48b1-8e57-597d9e94998a,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList l_e = new ArrayList(); -",212,1.0 +",212,1.0 2025-09-22T14:00:54.613908,[1],"def is_palindrome(text: str) -> bool: """""" Checks if given string is a palindrome @@ -50538,7 +50538,7 @@ class Problem { ",93.42389999801526,,Completions generated successfully. Ready for display in your IDE.,080bc018-5c5b-4171-a701-9daabf3c1d6a,1,deepseek-ai/deepseek-coder-1.3b-base," return False -",72,1.0 +",72,1.0 2025-09-22T14:00:54.768855,[1],"//Write a function that returns true if the given number is the multiplication of 3 prime numbers // and false otherwise. // Knowing that (a) is less then 100. @@ -50564,7 +50564,7 @@ function is_multiply_prime(a){ } }",155.10989999893354,,Completions generated successfully. Ready for display in your IDE.,478a40ff-aa0c-45af-9dc5-a840ebd8875a,1,deepseek-ai/deepseek-coder-1.3b-base," while(!isNaN(a)){ -",132,0.9257 +",132,0.9257 2025-09-22T14:00:54.868007,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50596,7 +50596,7 @@ class Problem { } }",99.0382999952999,,Completions generated successfully. Ready for display in your IDE.,4081eb53-a891-4300-b398-5ec6b3dd66ed,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:00:54.982429,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -50673,7 +50673,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",113.73189999721944,,Completions generated successfully. Ready for display in your IDE.,3e8bb3a6-1f67-4a3c-a905-8b8757d7ded4,1,deepseek-ai/deepseek-coder-1.3b-base," return None -",87,1.0 +",87,1.0 2025-09-22T14:00:55.277620,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50710,7 +50710,7 @@ class Problem { } }",295.0032000007923,,Completions generated successfully. Ready for display in your IDE.,e8817848-d9b7-43cd-91a2-6860684bb05c,1,deepseek-ai/deepseek-coder-1.3b-base," char c = paren_string.charAt(i); -",204,1.0 +",204,1.0 2025-09-22T14:00:55.634035,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50752,7 +50752,7 @@ class Problem { } }",356.8834000034258,,Completions generated successfully. Ready for display in your IDE.,08c00009-88df-4fcb-8ea8-a622e9f1f448,1,deepseek-ai/deepseek-coder-1.3b-base," int sm = (int)s.chars().filter(c -> Character.isLowerCase(c)).count(); -",329,1.0 +",329,1.0 2025-09-22T14:00:55.844169,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50790,7 +50790,7 @@ class Problem { } }",209.8755999977584,,Completions generated successfully. Ready for display in your IDE.,c0484343-991f-4989-b212-345ce0ba1508,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (chars[i] == '.') { -",191,0.9505 +",191,0.9505 2025-09-22T14:00:55.943894,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50810,7 +50810,7 @@ class Problem { }"," } }",98.99330000189366,,Completions generated successfully. Ready for display in your IDE.,0ed3158c-2778-498e-ae0d-d84c77a7c20c,1,deepseek-ai/deepseek-coder-1.3b-base," return results; -",81,1.0 +",81,1.0 2025-09-22T14:00:56.187236,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50875,7 +50875,7 @@ class Problem { } }",242.9688000047463,,Completions generated successfully. Ready for display in your IDE.,1aae2a67-0274-4b00-b33c-77dd4cba4c2d,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (currentGrade > 2.3f) { -",217,1.0 +",217,1.0 2025-09-22T14:00:56.585954,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -50925,7 +50925,7 @@ class Problem { }",399.07900000252994,,Completions generated successfully. Ready for display in your IDE.,e98d927a-1b40-4e7a-935c-2bea03d84c09,1,deepseek-ai/deepseek-coder-1.3b-base," if (year < 1900 && year > 2020) return valid_date; if (month == 2) { -",379,0.9664 +",379,0.9664 2025-09-22T14:00:57.173273,[1],"//Create a function that returns true if the last character // of a given string is an alphabetical character and is not // a part of a word, and false otherwise. @@ -50946,7 +50946,7 @@ function check_if_last_char_is_a_letter(txt){ }",586.2827999953879,,Completions generated successfully. Ready for display in your IDE.,aaa8ca71-59e7-40a9-8779-c408e4e80782,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i = 0 ;i 1 and is not a prime. // >>> largest_prime_factor(13195) // 29 @@ -50963,7 +50963,7 @@ function largest_prime_factor(n){ } }",},95.47249999741325,,Completions generated successfully. Ready for display in your IDE.,c7f3fb02-728d-4d64-857c-9725e97fe99f,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",72,0.9947 +",72,0.9947 2025-09-22T14:00:57.373806,[1],"//Evaluate whether the given number n can be written as the sum of exactly 4 positive even numbers // Example // >>> is_equal_to_sum_even(4) @@ -50992,7 +50992,7 @@ function is_equal_to_sum_even(n){ else{ return false; }",,105.00670000328682,,Completions generated successfully. Ready for display in your IDE.,19668dec-35ea-46f6-9e68-819f38aeff72,1,deepseek-ai/deepseek-coder-1.3b-base," -}",45,1.0 +}",45,1.0 2025-09-22T14:00:57.439557,[1],"//Given a string, find out how many distinct characters (regardless of case) does it consist of // >>> count_distinct_characters(""xyzXYZ"") // 3 @@ -51007,7 +51007,7 @@ function count_distinct_characters(string){ } } return result.length;",,65.53870000061579,,Completions generated successfully. Ready for display in your IDE.,069f1032-2a72-4ff8-9a03-7a0c9750b965,1,deepseek-ai/deepseek-coder-1.3b-base," -}",41,1.0 +}",41,1.0 2025-09-22T14:00:57.616861,[1],"from typing import List def can_arrange(arr: List[int]) -> int: @@ -51027,7 +51027,7 @@ def can_arrange(arr: List[int]) -> int: return ans ",177.1387999979197,,Completions generated successfully. Ready for display in your IDE.,9c8f917a-5d0c-407d-9ede-3956034fcdc6,1,deepseek-ai/deepseek-coder-1.3b-base," for i in range(len(arr)): -",154,1.0 +",154,1.0 2025-09-22T14:00:57.735679,[1],"from typing import List def find_max(words: List[str]) -> str: @@ -51053,7 +51053,7 @@ def find_max(words: List[str]) -> str: elif unique_num == max_num: if word < max_word: max_word = word",,118.44360000395682,,Completions generated successfully. Ready for display in your IDE.,5dfb6088-e1b4-48ca-9cb5-e94375adfe44,1,deepseek-ai/deepseek-coder-1.3b-base," - return max_word",93,0.9576 + return max_word",93,0.9576 2025-09-22T14:00:57.802146,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51070,7 +51070,7 @@ class Problem { public static String concatenate(ArrayList strings) { return strings.stream().collect(Collectors.joining()); }",,65.62539999868022,,Completions generated successfully. Ready for display in your IDE.,20f0f85b-d818-4242-a134-c370a3a6689a,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,0.7409 +}",42,0.7409 2025-09-22T14:00:57.961012,[1],"//You are given an array of two strings, both strings consist of open // parentheses '(' or close parentheses ')' only. // Your job is to check if it is possible to concatenate the two strings in @@ -51110,7 +51110,7 @@ function match_parens(lst){ return 'No'; }",158.69129999919096,,Completions generated successfully. Ready for display in your IDE.,032a3ebf-54e6-4904-be40-4341f513c2fb,1,deepseek-ai/deepseek-coder-1.3b-base," for (const p of s){ -",130,1.0 +",130,1.0 2025-09-22T14:00:58.055833,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51143,7 +51143,7 @@ class Problem { return new String(arr); } }",95.07959999609739,,Completions generated successfully. Ready for display in your IDE.,9f958e57-4823-41b3-a956-0ab2733d7f2f,1,deepseek-ai/deepseek-coder-1.3b-base," - ",43,1.0 + ",43,1.0 2025-09-22T14:00:58.152878,[1],"def words_in_sentence(sentence: str) -> str: """""" You are given a string representing a sentence, @@ -51177,7 +51177,7 @@ class Problem { prime_words.append(word) return ' '.join(prime_words)",96.71569999773055,,Completions generated successfully. Ready for display in your IDE.,9efe3a3f-a8dd-4611-b97f-31228aaab91a,1,deepseek-ai/deepseek-coder-1.3b-base," return False -",72,1.0 +",72,1.0 2025-09-22T14:00:58.227923,[1],"//Return true is array elements are monotonically increasing or decreasing. // >>> monotonic([1, 2, 4, 20]) // true @@ -51204,7 +51204,7 @@ function monotonic(l){ return true; } ",74.68030000018189,,Completions generated successfully. Ready for display in your IDE.,36ea359f-d2b8-4665-9063-fe7f3c44ead4,1,deepseek-ai/deepseek-coder-1.3b-base,"else{ -",52,0.981 +",52,0.981 2025-09-22T14:00:58.373599,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -51248,7 +51248,7 @@ function histogram(test){ } ",145.3814999986207,,Completions generated successfully. Ready for display in your IDE.,6656e8fb-198a-43da-882f-25d2a0d01ef3,1,deepseek-ai/deepseek-coder-1.3b-base," max = count[key]; -",120,1.0 +",120,1.0 2025-09-22T14:00:58.491001,[1],"//Given an integer. return an array that has the number of even and odd digits respectively. // Example: // >>> even_odd_count(-12) @@ -51269,7 +51269,7 @@ function even_odd_count(num){ } ",117.6019999984419,,Completions generated successfully. Ready for display in your IDE.,67c96793-17a8-41e2-bac4-a5d408a43297,1,deepseek-ai/deepseek-coder-1.3b-base," var odds = 0; -",93,1.0 +",93,1.0 2025-09-22T14:00:58.573893,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51290,7 +51290,7 @@ class Problem { public static boolean rightAngleTriangle(long a, long b, long c) { return a*a + b*b == c*c || b*b + c*c == a*a || c*c + a*a == b*b;",},82.65559999563266,,Completions generated successfully. Ready for display in your IDE.,36e1cb35-93be-4606-98e5-65a2c8ecd8bf,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:00:58.660665,[1],"//Given an array of positive integers x. return a sorted array of all // elements that hasn't any even digit. // Note: Returned array should be sorted in increasing order. @@ -51310,7 +51310,7 @@ function unique_digits(x){ ",86.22900000045775,,Completions generated successfully. Ready for display in your IDE.,d6cc2228-a89e-41e5-8b61-2bea146db424,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:00:58.783722,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51349,7 +51349,7 @@ class Problem { } }",123.06889999308623,,Completions generated successfully. Ready for display in your IDE.,90342feb-ff48-4fab-b68b-d43ccaf709c2,1,deepseek-ai/deepseek-coder-1.3b-base," index = 0; -",98,0.9481 +",98,0.9481 2025-09-22T14:00:59.008011,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51393,7 +51393,7 @@ class Problem { } }",224.03249999479158,,Completions generated successfully. Ready for display in your IDE.,d24d8f1a-c511-4dbb-b4ba-bcda57d83e1f,1,deepseek-ai/deepseek-coder-1.3b-base," else if (op.get(i).equals(""**"")) -",196,1.0 +",196,1.0 2025-09-22T14:00:59.182910,[1],"//Return array of prime factors of given integer in the order from smallest to largest. // Each of the factors should be arrayed number of times corresponding to how many times it appeares in factorization. // Input number should be equal to the product of all factors @@ -51416,7 +51416,7 @@ function factorize(n){ } ",174.67889999534236,,Completions generated successfully. Ready for display in your IDE.,f4dba343-88bb-4545-97ca-0dcf7342d227,1,deepseek-ai/deepseek-coder-1.3b-base," if ((n % factor === 0)) { -",147,0.9844 +",147,0.9844 2025-09-22T14:00:59.292516,[1],"from typing import List def select_words(s: str, n: int) -> List[str]: @@ -51448,7 +51448,7 @@ def select_words(s: str, n: int) -> List[str]: return result ",108.91020000417484,,Completions generated successfully. Ready for display in your IDE.,ca524985-3943-4885-b24d-5179fb826b8a,1,deepseek-ai/deepseek-coder-1.3b-base," result = [] -",83,1.0 +",83,1.0 2025-09-22T14:00:59.574479,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51490,7 +51490,7 @@ class Problem { } }",281.52959999715677,,Completions generated successfully. Ready for display in your IDE.,9b4f3dea-4e24-4f06-b545-a21783d45a19,1,deepseek-ai/deepseek-coder-1.3b-base," if (s3.charAt(i) == '(') c1 += 1; -",255,1.0 +",255,1.0 2025-09-22T14:00:59.680152,[1],"from typing import List def odd_count(lst: List[str]) -> List[str]: @@ -51512,7 +51512,7 @@ def odd_count(lst: List[str]) -> List[str]: return output ",106.06429999461398,,Completions generated successfully. Ready for display in your IDE.,a7fe716d-868f-4141-8105-4d483a104011,1,deepseek-ai/deepseek-coder-1.3b-base," count += 1 -",82,1.0 +",82,1.0 2025-09-22T14:00:59.814479,[1],"from typing import List def count_nums(arr: List[int]) -> int: @@ -51542,7 +51542,7 @@ def count_nums(arr: List[int]) -> int: return n ",133.97819999954663,,Completions generated successfully. Ready for display in your IDE.,03ccb214-e37c-46e8-b562-2140a5e45dd1,1,deepseek-ai/deepseek-coder-1.3b-base," s += int(j) -",112,1.0 +",112,1.0 2025-09-22T14:00:59.979910,[1],"//Given a string representing a space separated lowercase letters, return an object // of the letter with the most repetition and containing the corresponding count. // If several letters have the same occurrence, return all of them. @@ -51586,7 +51586,7 @@ function histogram(test){ } ",165.2570000005653,,Completions generated successfully. Ready for display in your IDE.,053a9368-0f61-4f8b-b337-5ac0dd17534a,1,deepseek-ai/deepseek-coder-1.3b-base," //console.log(lst) -",141,0.8655 +",141,0.8655 2025-09-22T14:01:00.132407,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51664,7 +51664,7 @@ class Problem { } }",152.25910000299336,,Completions generated successfully. Ready for display in your IDE.,4c123407-e258-40cc-b0f6-51aa269202a2,1,deepseek-ai/deepseek-coder-1.3b-base," num -= 1; -",112,0.9202 +",112,0.9202 2025-09-22T14:01:00.299853,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51702,7 +51702,7 @@ class Problem { } }",166.62360000191256,,Completions generated successfully. Ready for display in your IDE.,1dc27089-f598-4b0b-bd7d-a2a2dbe253a4,1,deepseek-ai/deepseek-coder-1.3b-base," sum1 += s.length(); -",138,1.0 +",138,1.0 2025-09-22T14:01:00.475545,[1],"//Implement the function f that takes n as a parameter, // and returns an array of size n, such that the value of the element at index i is the factorial of i if i is even // or the sum of numbers from 1 to i otherwise. @@ -51732,7 +51732,7 @@ function f(n){ return result; }",175.96559999947203,,Completions generated successfully. Ready for display in your IDE.,a43e92f5-8f9e-4c70-8e54-92d95a4f954a,1,deepseek-ai/deepseek-coder-1.3b-base," // your code here var result = []; -",153,0.9316 +",153,0.9316 2025-09-22T14:01:00.616898,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51777,7 +51777,7 @@ class Problem { } }",140.93850000062957,,Completions generated successfully. Ready for display in your IDE.,5f6d2ba9-038f-4543-9560-4f775a16f5e4,1,deepseek-ai/deepseek-coder-1.3b-base," boolean isPrime; -",110,0.9818 +",110,0.9818 2025-09-22T14:01:00.814384,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51819,7 +51819,7 @@ class Problem { } }",197.55860000441317,,Completions generated successfully. Ready for display in your IDE.,08f30600-fe16-4c3b-9b47-3ec62b390d2b,1,deepseek-ai/deepseek-coder-1.3b-base," sb.append(s.charAt(i)); -",173,1.0 +",173,1.0 2025-09-22T14:01:01.224546,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -51844,7 +51844,7 @@ class Problem { } }",409.49080000427784,,Completions generated successfully. Ready for display in your IDE.,14b1b989-eb94-4c6f-82fd-84cd882af18c,1,deepseek-ai/deepseek-coder-1.3b-base," ret = ((double)(l.get(size / 2 - 1) + l.get(size / 2))) / 2d; -",385,0.9886 +",385,0.9886 2025-09-22T14:01:01.484092,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -51921,7 +51921,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",259.72919999912847,,Completions generated successfully. Ready for display in your IDE.,e4a1d9f0-e78d-4f6c-a09f-c3663a9c49e4,1,deepseek-ai/deepseek-coder-1.3b-base," elif isinstance(a, str) and isinstance(b, float): -",232,0.975 +",232,0.975 2025-09-22T14:01:01.887819,[1],"from typing import List def sorted_list_sum(lst: List[str]) -> List[str]: @@ -51943,7 +51943,7 @@ def sorted_list_sum(lst: List[str]) -> List[str]: """""" ",,403.7968999982695,,Completions generated successfully. Ready for display in your IDE.,500ce5ae-5747-4150-997f-ce7d4c9ee77e,1,deepseek-ai/deepseek-coder-1.3b-base," lst = [i for i in lst if len(i) % 2 == 0] lst.sort() - return lst",380,0.9563 + return lst",380,0.9563 2025-09-22T14:01:01.932456,[1],"//You are given a string s. // if s[i] is a letter, reverse its case from lower to upper or vise versa, // otherwise keep it as it is. @@ -51970,7 +51970,7 @@ function solve(s){ } return s.split('').reverse().join(''); } -",44.17959999409504,,Completions generated successfully. Ready for display in your IDE.,999908d5-f8ab-45d0-bce7-393ad422b665,1,deepseek-ai/deepseek-coder-1.3b-base,,22,0.859 +",44.17959999409504,,Completions generated successfully. Ready for display in your IDE.,999908d5-f8ab-45d0-bce7-393ad422b665,1,deepseek-ai/deepseek-coder-1.3b-base,,22,0.859 2025-09-22T14:01:02.102067,[1],"//Input to this function is a string containing multiple groups of nested parentheses. Your goal is to // separate those group into separate strings and return the array of those. // Separate groups are balanced (each open brace is properly closed) and not nested within each other @@ -51996,7 +51996,7 @@ function separate_paren_groups(paren_string){ } ",169.48609999963082,,Completions generated successfully. Ready for display in your IDE.,1bed5597-7fd8-44bf-8925-1a05bfdeb4cc,1,deepseek-ai/deepseek-coder-1.3b-base," group += paren_string[i]; -",145,1.0 +",145,1.0 2025-09-22T14:01:02.216174,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52029,7 +52029,7 @@ class Problem { }"," } }",113.44200000166893,,Completions generated successfully. Ready for display in your IDE.,f3650218-b093-4377-be16-0aa1cb6c6943,1,deepseek-ai/deepseek-coder-1.3b-base," return sum; -",89,1.0 +",89,1.0 2025-09-22T14:01:02.477528,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52073,7 +52073,7 @@ class Problem { } }",261.61470000079134,,Completions generated successfully. Ready for display in your IDE.,d987235a-d757-4724-918b-033b86ea630c,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < word.length(); i++) { -",235,1.0 +",235,1.0 2025-09-22T14:01:03.177415,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52120,7 +52120,7 @@ class Problem { while (!Character.isLetter(arr[j])){ j--; } - Collections.reverse(",674,0.974 + Collections.reverse(",674,0.974 2025-09-22T14:01:03.297905,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52161,7 +52161,7 @@ class Problem { }"," } }",120.28180000197608,,Completions generated successfully. Ready for display in your IDE.,6808e0fb-1a1d-4daf-8d12-a702178f5b0f,1,deepseek-ai/deepseek-coder-1.3b-base," return res; -",92,1.0 +",92,1.0 2025-09-22T14:01:03.409886,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -52229,7 +52229,7 @@ function do_algebra(operator, operand){ } ",111.61980000179028,,Completions generated successfully. Ready for display in your IDE.,21d7fbbe-185c-483a-8d58-273c03bc0b2d,1,deepseek-ai/deepseek-coder-1.3b-base," break; -",82,1.0 +",82,1.0 2025-09-22T14:01:03.500425,[1],"//Write a function that returns true if the object q will fly, and false otherwise. // The object q will fly if it's balanced (it is a palindromic array) and the sum of its elements is less than or equal the maximum possible weight w. // Example: @@ -52258,7 +52258,7 @@ function will_it_fly(q, w){ return balanced && sum <= w; }",90.69269999599783,,Completions generated successfully. Ready for display in your IDE.,fcb6c466-d519-4409-8f20-fab8fd06b136,1,deepseek-ai/deepseek-coder-1.3b-base," } -",64,1.0 +",64,1.0 2025-09-22T14:01:03.725009,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -52325,7 +52325,7 @@ function numerical_letter_grade(grades){ return grades_array; }",223.75550000288058,,Completions generated successfully. Ready for display in your IDE.,1dc94bd3-360f-458e-87bb-1eb14ae416a0,1,deepseek-ai/deepseek-coder-1.3b-base," else if (grades[i] > 0.7){ -",191,1.0 +",191,1.0 2025-09-22T14:01:03.906841,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52365,7 +52365,7 @@ class Problem { } }",182.10790000011912,,Completions generated successfully. Ready for display in your IDE.,36e4da56-47e7-4341-b851-01c4c4702212,1,deepseek-ai/deepseek-coder-1.3b-base," long z = x.get(i); -",149,1.0 +",149,1.0 2025-09-22T14:01:04.014853,[1],"//From an array of integers, remove all elements that occur more than once. // Keep order of elements left the same as in the input. // >>> remove_duplicates([1, 2, 3, 2, 4]) @@ -52379,7 +52379,7 @@ function remove_duplicates(numbers){ });","} ",106.98660000343807,,Completions generated successfully. Ready for display in your IDE.,aca522a1-c2d5-4cbb-9ff4-bbe60b59efb6,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",83,0.9937 +",83,0.9937 2025-09-22T14:01:04.176910,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52425,7 +52425,7 @@ class Problem { }",162.44410000217613,,Completions generated successfully. Ready for display in your IDE.,dbe8db40-bb51-43b4-978c-b883bc1d1779,1,deepseek-ai/deepseek-coder-1.3b-base," long sum = 0; -",133,0.9065 +",133,0.9065 2025-09-22T14:01:04.427333,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52460,7 +52460,7 @@ class Problem { } }",249.66870000207564,,Completions generated successfully. Ready for display in your IDE.,543220eb-4459-44e9-b82e-d2609d322cd1,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < sz; i++) { -",222,1.0 +",222,1.0 2025-09-22T14:01:04.712905,[1],"//brackets is a string of ""("" and "")"". // return true if every opening bracket has a corresponding closing bracket. // >>> correct_bracketing(""("") @@ -52493,7 +52493,7 @@ function correct_bracketing(brackets){ }",285.683499998413,,Completions generated successfully. Ready for display in your IDE.,89cfc957-0cd8-409f-bd07-1c58b93bbe3e,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i = 0; i < brackets.length; i++ ){ -",260,0.9907 +",260,0.9907 2025-09-22T14:01:04.790160,[1],"//You are given a string s. // Your task is to check if the string is hapjs or not. // A string is hapjs if its length is at least 3 and every 3 consecutive letters are distinct @@ -52521,7 +52521,7 @@ function is_happy(s){ } } return happy;",,77.19179999548942,,Completions generated successfully. Ready for display in your IDE.,3ce67ca6-b5fa-4de2-9022-e8ac6de894b0,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,0.9689 +}",49,0.9689 2025-09-22T14:01:04.877622,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52553,7 +52553,7 @@ class Problem { } return isPalindrome;",},87.24240000447026,,Completions generated successfully. Ready for display in your IDE.,0062b07f-623b-497a-aad1-536c1b15d712,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:01:05.123002,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52593,7 +52593,7 @@ class Problem { long left = Math.max(0, remaining - need);"," } }",245.2248999979929,,Completions generated successfully. Ready for display in your IDE.,a4e0a70d-6881-4295-8bc3-38e5b7ff3dca,1,deepseek-ai/deepseek-coder-1.3b-base," return new ArrayList<>(Arrays.asList(total, left)); -",214,1.0 +",214,1.0 2025-09-22T14:01:05.334148,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -52632,7 +52632,7 @@ function valid_date(date){ } ",210.48380000138422,,Completions generated successfully. Ready for display in your IDE.,86f3ecca-b862-4eae-b5b7-dabba5d1041e,1,deepseek-ai/deepseek-coder-1.3b-base," var month = parseInt(date_arr[0]); -",183,1.0 +",183,1.0 2025-09-22T14:01:05.443660,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52685,7 +52685,7 @@ class Problem { newArr.add((long)minIndex); return newArr;",},109.80600000038976,,Completions generated successfully. Ready for display in your IDE.,0a6b10df-75ca-4d2d-97b3-b404fc443c87,1,deepseek-ai/deepseek-coder-1.3b-base," } -",80,1.0 +",80,1.0 2025-09-22T14:01:06.136464,[1],"def encrypt(s: str) -> str: """"""Create a function encrypt that takes a string as an argument and returns a string encrypted with the alphabet being rotated. @@ -52707,7 +52707,7 @@ class Problem { ",692.3715000011725,,Completions generated successfully. Ready for display in your IDE.,dfd34d8a-295a-4df9-ba4b-6857c2949a89,1,deepseek-ai/deepseek-coder-1.3b-base," if ord(i) >= 65 and ord(i) <= 90: #if uppercase output_string += chr((ord(i)-23-1)) - elif ord(i) >= 97 and ord(i) <= 122: #if lower",668,0.9623 + elif ord(i) >= 97 and ord(i) <= 122: #if lower",668,0.9623 2025-09-22T14:01:06.243469,[1],"from typing import List def unique_digits(x: List[int]) -> List[int]: @@ -52733,7 +52733,7 @@ def unique_digits(x: List[int]) -> List[int]: return sorted(x) ",107.02060000039637,,Completions generated successfully. Ready for display in your IDE.,999cf529-42eb-4268-a1fb-5d0075dfa397,1,deepseek-ai/deepseek-coder-1.3b-base," temp=i -",84,1.0 +",84,1.0 2025-09-22T14:01:06.307483,[1],"//Given a positive integer n, you have to make a pile of n levels of stones. // The first level has n stones. // The number of stones in the next level is: @@ -52750,7 +52750,7 @@ function make_a_pile(n){ arr.push(n + (i * 2)); } return arr;",,63.82099999609636,,Completions generated successfully. Ready for display in your IDE.,264eff55-8d61-4317-b70f-dcfc89b98594,1,deepseek-ai/deepseek-coder-1.3b-base," -}",42,1.0 +}",42,1.0 2025-09-22T14:01:06.381880,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52785,7 +52785,7 @@ class Problem { return (lst1.stream().allMatch(x -> x % 2 == 0)) ? ""YES"" : ""NO""; } }",74.14750000316417,,Completions generated successfully. Ready for display in your IDE.,3456bb7c-8277-4922-9fac-7dcfe77679bf,1,deepseek-ai/deepseek-coder-1.3b-base," - ",49,0.8545 + ",49,0.8545 2025-09-22T14:01:06.480167,[1],"from typing import List def odd_count(lst: List[str]) -> List[str]: @@ -52807,7 +52807,7 @@ def odd_count(lst: List[str]) -> List[str]: output.append(f""the number of odd elements {count}n the str{count}ng {count} of the {count}nput."") return output ",97.84819999913452,,Completions generated successfully. Ready for display in your IDE.,d6e05da7-ce47-40a0-ad89-d9e30085fa1a,1,deepseek-ai/deepseek-coder-1.3b-base," output = [] -",73,1.0 +",73,1.0 2025-09-22T14:01:06.566532,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52831,7 +52831,7 @@ class Problem { } }",86.44570000615204,,Completions generated successfully. Ready for display in your IDE.,058eb06b-20a7-43d3-82a9-d036e6ce8a7e,1,deepseek-ai/deepseek-coder-1.3b-base," } -",63,1.0 +",63,1.0 2025-09-22T14:01:06.691846,[1],"from typing import Tuple def intersection(interval1: Tuple[int, int], interval2: Tuple[int, int]) -> str: @@ -52862,7 +52862,7 @@ def intersection(interval1: Tuple[int, int], interval2: Tuple[int, int]) -> str: if start > end:"," return ""YES"" if end - start == 2 else ""NO"" ",125.12270000297576,,Completions generated successfully. Ready for display in your IDE.,e383f497-4dce-4ec7-b8a8-68de2da115b8,1,deepseek-ai/deepseek-coder-1.3b-base," return ""NO"" -",99,1.0 +",99,1.0 2025-09-22T14:01:06.906968,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52927,7 +52927,7 @@ class Problem { } }",215.17989999847487,,Completions generated successfully. Ready for display in your IDE.,3b8c0836-7b6e-4ffe-bd39-d365c0ba2c62,1,deepseek-ai/deepseek-coder-1.3b-base," float currentGrade = grades.get(i); -",186,1.0 +",186,1.0 2025-09-22T14:01:07.479625,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -52944,7 +52944,7 @@ class Problem { public static ArrayList filterIntegers(ArrayList values) { "," } }",572.0447999992757,,Completions generated successfully. Ready for display in your IDE.,ed3e5779-1ddf-447b-8016-51d031bbda24,1,deepseek-ai/deepseek-coder-1.3b-base," return (ArrayList)(values).stream().filter(value -> value instanceof Long || value instanceof Integer).map(value -> ((Number)value).longValue()).collect(Collectors.toCollection(ArrayList::new)); -",546,0.9899 +",546,0.9899 2025-09-22T14:01:07.581743,[1],"//You are given an array of integers. // Write a function next_smallest() that returns the 2nd smallest element of the array. // Return undefined if there is no such element. @@ -52971,7 +52971,7 @@ function next_smallest(lst){ return second_smallest; }",101.89799999352545,,Completions generated successfully. Ready for display in your IDE.,9a9098b6-5192-4950-9f65-6a6400449c1d,1,deepseek-ai/deepseek-coder-1.3b-base," return undefined; -",81,1.0 +",81,1.0 2025-09-22T14:01:07.656664,[1],"//Filter given array of any jsthon values only for integers // >>> filter_integers([""a"", 3.14, 5]) // [5] @@ -52986,7 +52986,7 @@ function filter_integers(values){ } ",74.33250000030966,,Completions generated successfully. Ready for display in your IDE.,59504b74-87e5-4a34-a23f-dd82b5933071,1,deepseek-ai/deepseek-coder-1.3b-base," }); -",53,1.0 +",53,1.0 2025-09-22T14:01:07.846722,[1],"from typing import List def parse_nested_parens(paren_string: str) -> List[int]: @@ -53009,7 +53009,7 @@ def parse_nested_parens(paren_string: str) -> List[int]: return result ",190.28939999407157,,Completions generated successfully. Ready for display in your IDE.,6fe671be-e877-4f72-a542-57a0f9482e3c,1,deepseek-ai/deepseek-coder-1.3b-base," for parens in paren_string.split(): -",167,0.9899 +",167,0.9899 2025-09-22T14:01:07.940209,[1],"def is_palindrome(text: str) -> bool: """""" Checks if given string is a palindrome @@ -53027,7 +53027,7 @@ def parse_nested_parens(paren_string: str) -> List[int]: if text[i] != text[-i - 1]: return False"," ",93.044799999916,,Completions generated successfully. Ready for display in your IDE.,699ac823-32b6-45ac-944b-3f198f15e19f,1,deepseek-ai/deepseek-coder-1.3b-base," - return True",62,1.0 + return True",62,1.0 2025-09-22T14:01:08.034848,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53067,7 +53067,7 @@ class Problem { } }",94.07780000037747,,Completions generated successfully. Ready for display in your IDE.,036d2584-4cb6-4cd0-9038-6da7ca23dc89,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:01:08.101050,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53107,7 +53107,7 @@ class Problem { return res; } }",66.00529999559512,,Completions generated successfully. Ready for display in your IDE.,3a9d66ee-aca1-414d-8140-ecafbf26c42d,1,deepseek-ai/deepseek-coder-1.3b-base," -",39,1.0 +",39,1.0 2025-09-22T14:01:08.670176,[1],"//Given a string, find out how many distinct characters (regardless of case) does it consist of // >>> count_distinct_characters(""xyzXYZ"") // 3 @@ -53125,7 +53125,7 @@ function count_distinct_characters(string){ console.log('The number of unique letters in the given word is: ' + arr.length); //loop through array and check to see if letter already exists within our new empty object/array called ""result"" -",543,0.8458 +",543,0.8458 2025-09-22T14:01:08.737618,[1],"//You're a hungry rabbit, and you already have eaten a certain number of carrots, // but now you need to eat more carrots to complete the day's meals. // you should return an array of [ total number of eaten carrots after your meals, @@ -53155,7 +53155,7 @@ function count_distinct_characters(string){ function eat(number, need, remaining){ if (need > remaining) return [number + remaining, 0]; else return [number + need, remaining - need];",,66.6553000046406,,Completions generated successfully. Ready for display in your IDE.,6e5e41ce-d265-4e37-8a13-62b5e380a480,1,deepseek-ai/deepseek-coder-1.3b-base," -}",43,1.0 +}",43,1.0 2025-09-22T14:01:08.826528,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53185,7 +53185,7 @@ class Problem { factors.add(n); return factors;",},88.88699999806704,,Completions generated successfully. Ready for display in your IDE.,6782bbfb-37fd-4a6e-a801-7598b4469bab,1,deepseek-ai/deepseek-coder-1.3b-base," } -",66,1.0 +",66,1.0 2025-09-22T14:01:09.047889,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53220,7 +53220,7 @@ class Problem { } }",220.8258999962709,,Completions generated successfully. Ready for display in your IDE.,7e00b050-88c3-4c86-bc7c-5206de912b84,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (i % 4 != 0) { -",196,0.9856 +",196,0.9856 2025-09-22T14:01:09.141745,[1],"def strlen(string: str) -> int: """""" Return length of given string >>> strlen('') @@ -53233,7 +53233,7 @@ class Problem { return length ",94.18649999861373,,Completions generated successfully. Ready for display in your IDE.,87a0c986-b99b-4aae-81f4-edd13fed96dd,1,deepseek-ai/deepseek-coder-1.3b-base," length = 0 -",72,1.0 +",72,1.0 2025-09-22T14:01:09.229360,[1],"//Task // Write a function that takes a string as input and returns the sum of the upper characters only' // ASCII codes. @@ -53258,7 +53258,7 @@ function digitSum(s){ return sum; }",87.56410000205506,,Completions generated successfully. Ready for display in your IDE.,e4e03568-56ea-4287-af2a-320c785305ef,1,deepseek-ai/deepseek-coder-1.3b-base," } -",65,1.0 +",65,1.0 2025-09-22T14:01:09.434721,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53297,7 +53297,7 @@ class Problem { } }",205.16079999651993,,Completions generated successfully. Ready for display in your IDE.,b9f714de-6daa-4f59-9269-abe83535700c,1,deepseek-ai/deepseek-coder-1.3b-base," long currSum = nums.get(i); -",181,1.0 +",181,1.0 2025-09-22T14:01:09.544566,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53337,7 +53337,7 @@ class Problem { } }",109.22539999592118,,Completions generated successfully. Ready for display in your IDE.,e2f4641e-2987-4328-b26a-f30ff73cdc58,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",84,1.0 +",84,1.0 2025-09-22T14:01:09.879978,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53354,7 +53354,7 @@ class Problem { public static ArrayList getPositive(ArrayList l) { "," } }",334.9744000006467,,Completions generated successfully. Ready for display in your IDE.,d22540f4-c490-4a30-bdb9-711e5f24ecb0,1,deepseek-ai/deepseek-coder-1.3b-base," return l.stream().filter(i -> i > 0).collect(Collectors.toCollection(ArrayList::new)); -",311,0.9679 +",311,0.9679 2025-09-22T14:01:09.974916,[1],"//You have to write a function which validates a given date string and // returns true if the date is valid otherwise false. // The date is valid if all of the following rules are satisfied: @@ -53393,7 +53393,7 @@ function valid_date(date){ } ",95.6240000014077,,Completions generated successfully. Ready for display in your IDE.,c986257e-2000-4360-a4c2-58d87dbcbed6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",69,1.0 +",69,1.0 2025-09-22T14:01:10.061243,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53429,7 +53429,7 @@ class Problem { } return max; }",,85.43840000493219,,Completions generated successfully. Ready for display in your IDE.,58392109-a47f-44e3-a180-30c2d88dda1c,1,deepseek-ai/deepseek-coder-1.3b-base," -}",48,1.0 +}",48,1.0 2025-09-22T14:01:10.180758,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53459,7 +53459,7 @@ class Problem { return res; } }",119.02420000114944,,Completions generated successfully. Ready for display in your IDE.,ce3f3acc-907a-48b7-a786-bcfab55ab8bc,1,deepseek-ai/deepseek-coder-1.3b-base," int res = 0; -",95,1.0 +",95,1.0 2025-09-22T14:01:10.358228,[1],"from typing import List def count_up_to(n: int) -> List[int]: @@ -53491,7 +53491,7 @@ def count_up_to(n: int) -> List[int]: return result ",177.94470000080764,,Completions generated successfully. Ready for display in your IDE.,7bf88ae6-d7da-4b89-9702-85c5999eefe4,1,deepseek-ai/deepseek-coder-1.3b-base," for divisor in range(2, number): -",154,0.9497 +",154,0.9497 2025-09-22T14:01:10.465688,[1],"from typing import List def factorize(n: int) -> List[int]: @@ -53513,7 +53513,7 @@ def factorize(n: int) -> List[int]: return factors ",107.2379999968689,,Completions generated successfully. Ready for display in your IDE.,165d5b13-9109-465d-a289-1ea96c4a9ba1,1,deepseek-ai/deepseek-coder-1.3b-base," i = 2 -",83,1.0 +",83,1.0 2025-09-22T14:01:10.723745,[1],"//This function takes two positive numbers x and y and returns the // biggest even integer number that is in the range [x, y] inclusive. If // there's no such number, then the function should return -1. @@ -53531,7 +53531,7 @@ function choose_num(x, y){ } ",257.0287000053213,,Completions generated successfully. Ready for display in your IDE.,4321cc9f-4e79-452d-945b-ffd9995af3f2,1,deepseek-ai/deepseek-coder-1.3b-base," for (var i=y-1; i>=x+1; --i ){ -",235,0.9691 +",235,0.9691 2025-09-22T14:01:10.818587,[1],"def prime_length(string: str) -> bool: """"""Write a function that takes a string and returns True if the string length is a prime number or False otherwise @@ -53556,7 +53556,7 @@ function choose_num(x, y){ ",94.68709999782732,,Completions generated successfully. Ready for display in your IDE.,bf1f519e-9490-4757-ad56-45a9842e0638,1,deepseek-ai/deepseek-coder-1.3b-base," else: -",71,1.0 +",71,1.0 2025-09-22T14:01:10.916618,[1],"from typing import List def monotonic(l: List[int]) -> bool: @@ -53571,7 +53571,7 @@ def monotonic(l: List[int]) -> bool: if l == sorted(l) or l == sorted(l, reverse=True):"," return False ",98.06229999958305,,Completions generated successfully. Ready for display in your IDE.,6662215f-5e49-4775-a76d-94096a819f81,1,deepseek-ai/deepseek-coder-1.3b-base," return True -",73,1.0 +",73,1.0 2025-09-22T14:01:10.984543,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -53638,7 +53638,7 @@ function do_algebra(operator, operand){ return answer; } ",67.97489999735262,,Completions generated successfully. Ready for display in your IDE.,a973bafe-43c5-429b-af0b-ae35919d04b6,1,deepseek-ai/deepseek-coder-1.3b-base," -",41,1.0 +",41,1.0 2025-09-22T14:01:11.124291,[1],"from typing import List def count_nums(arr: List[int]) -> int: @@ -53668,7 +53668,7 @@ def count_nums(arr: List[int]) -> int: return n ",139.78920000226935,,Completions generated successfully. Ready for display in your IDE.,379f8a7b-1cf8-4c32-bf3d-f23c00b308a0,1,deepseek-ai/deepseek-coder-1.3b-base," for i in arr: -",103,1.0 +",103,1.0 2025-09-22T14:01:11.240065,[1],"from typing import List def add_elements(arr: List[int], k: int) -> int: @@ -53690,7 +53690,7 @@ def add_elements(arr: List[int], k: int) -> int: if num < 100: total += num",,114.96530000295024,,Completions generated successfully. Ready for display in your IDE.,ea328161-1d37-4c3c-9be2-99ac882eebdd,1,deepseek-ai/deepseek-coder-1.3b-base," - return total",93,0.9648 + return total",93,0.9648 2025-09-22T14:01:11.447194,[1],"def valid_date(date: str) -> bool: """"""You have to write a function which validates a given date string and returns True if the date is valid otherwise False. @@ -53727,7 +53727,7 @@ def add_elements(arr: List[int], k: int) -> int: return False return True",207.35349999449681,,Completions generated successfully. Ready for display in your IDE.,1fe384cc-add3-4ce4-b14d-e80ac6de50b0,1,deepseek-ai/deepseek-coder-1.3b-base," if (len(date_list) != 3): -",182,0.9541 +",182,0.9541 2025-09-22T14:01:11.584281,[1],"from typing import List def pluck(arr: List[int]) -> List[int]: @@ -53770,7 +53770,7 @@ def pluck(arr: List[int]) -> List[int]: return [min(evens), arr.index(min(evens))] ",136.4508000042406,,Completions generated successfully. Ready for display in your IDE.,561b33eb-e3d6-4c86-823a-c88b28e1435d,1,deepseek-ai/deepseek-coder-1.3b-base," if not evens: -",111,1.0 +",111,1.0 2025-09-22T14:01:11.665231,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53837,7 +53837,7 @@ class Problem { } } }",81.22330000333022,,Completions generated successfully. Ready for display in your IDE.,0b21cf17-3782-4bb8-a13d-b9b56e486471,1,deepseek-ai/deepseek-coder-1.3b-base," -",52,1.0 +",52,1.0 2025-09-22T14:01:12.390484,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -53922,7 +53922,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i return None if __name__ == ""__main__"": - import",694,0.9848 + import",694,0.9848 2025-09-22T14:01:12.586672,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -53954,7 +53954,7 @@ class Problem { } }",196.6251000048942,,Completions generated successfully. Ready for display in your IDE.,4002a85c-2ac7-412c-a9ab-677cb063f2b4,1,deepseek-ai/deepseek-coder-1.3b-base," out.add(numbers.get(i)); -",172,1.0 +",172,1.0 2025-09-22T14:01:12.629980,[1],"from typing import List def remove_duplicates(numbers: List[int]) -> List[int]: @@ -53966,7 +53966,7 @@ def remove_duplicates(numbers: List[int]) -> List[int]: numbers.sort()"," return numbers ",42.70679999899585,,Completions generated successfully. Ready for display in your IDE.,c0e19921-e979-4183-ae9d-2aef1e666ff9,1,deepseek-ai/deepseek-coder-1.3b-base," -",23,0.6808 +",23,0.6808 2025-09-22T14:01:12.770161,[1],"from typing import List def match_parens(lst: List[str]) -> str: @@ -54002,7 +54002,7 @@ def match_parens(lst: List[str]) -> str: return 'No' ",139.77670000167564,,Completions generated successfully. Ready for display in your IDE.,90625d3d-8585-45dd-b93a-4cad6d81a7a1,1,deepseek-ai/deepseek-coder-1.3b-base," if c == '(': -",116,1.0 +",116,1.0 2025-09-22T14:01:13.500698,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54072,7 +54072,7 @@ class Problem { private static void main(String[] args){ System.out.println(""\n\n"" + ""Problem 6: Grading Algorithm"" + ""\n\n""); System.out.print(""Input: ""); -",701,0.9441 +",701,0.9441 2025-09-22T14:01:13.615643,[1],"//Given an object, return true if all keys are strings in lower // case or all keys are strings in upper case, else return false. // The function should return false is the given object is empty. @@ -54121,7 +54121,7 @@ function check_dict_case(dict){ return true; }",115.10759999509901,,Completions generated successfully. Ready for display in your IDE.,5dbd9e2c-a239-4d1c-8dfc-3466f9b2cf5a,1,deepseek-ai/deepseek-coder-1.3b-base," return false; -",87,1.0 +",87,1.0 2025-09-22T14:01:13.701241,[1],"//Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. // >>> fizz_buzz(50) // 0 @@ -54143,7 +54143,7 @@ function fizz_buzz(n){ } ",84.83040000282926,,Completions generated successfully. Ready for display in your IDE.,e2eeaa3a-7e5d-4558-b431-59ac473d51e6,1,deepseek-ai/deepseek-coder-1.3b-base," } -",62,1.0 +",62,1.0 2025-09-22T14:01:13.795441,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54176,7 +54176,7 @@ class Problem { } }",94.64129999832949,,Completions generated successfully. Ready for display in your IDE.,64b8e2b1-5b4e-46ee-9d24-e7b8cce8425c,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:01:13.942838,[1],"from typing import List def exchange(lst1: List[int], lst2: List[int]) -> str: @@ -54204,7 +54204,7 @@ def exchange(lst1: List[int], lst2: List[int]) -> str: else: return ""YES""",146.91349999338854,,Completions generated successfully. Ready for display in your IDE.,c9d134c1-d498-4590-a10c-9e1247e08d32,1,deepseek-ai/deepseek-coder-1.3b-base," for x in lst1: -",122,1.0 +",122,1.0 2025-09-22T14:01:14.642136,[1],"//Out of array of strings, return the longest one. Return the first one in case of multiple // strings of the same length. Return undefined in case the input array is empty. // >>> longest([]) @@ -54225,7 +54225,7 @@ function longest(strings){ } ",699.1903000016464,,Completions generated successfully. Ready for display in your IDE.,5e700835-9975-4ee8-828c-e0333388f418,1,deepseek-ai/deepseek-coder-1.3b-base," else if ((strings[i] === '') && !isNaN((parseInt('' + strings)))){ //if string contains only numbers and no letters or symbols then it will be falsey so we can use this to check that as well: console.log(""string was a number""); -",676,0.8404 +",676,0.8404 2025-09-22T14:01:14.722959,[1],"//This function takes an array l and returns an array l' such that // l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal // to the values of the corresponding indicies of l, but sorted. @@ -54254,7 +54254,7 @@ function sort_third(l){ } return x; }",80.64509999530856,,Completions generated successfully. Ready for display in your IDE.,38c33ff2-9cd5-474e-b836-bc8201c7677c,1,deepseek-ai/deepseek-coder-1.3b-base,"else{ -",58,1.0 +",58,1.0 2025-09-22T14:01:15.403517,[1],"//You have been tasked to write a function that receives // a hexadecimal number as a string and counts the number of hexadecimal // digits that are primes (prime number, or a prime, is a natural number @@ -54286,7 +54286,7 @@ function hex_key(num){ return result; }",680.4918999987422,,Completions generated successfully. Ready for display in your IDE.,f0d7783e-4341-44cd-bd70-0da9d70a69ca,1,deepseek-ai/deepseek-coder-1.3b-base," //array with all possible values in order from lowest value upto highest one const primes = [2, 3, 5, 7]; // array containing only those which are divisible by these three numbers -> 2, 3, 5 & 7 respectively -",655,0.888 +",655,0.888 2025-09-22T14:01:15.500315,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54322,7 +54322,7 @@ class Problem { } }",95.77540000464069,,Completions generated successfully. Ready for display in your IDE.,0459e577-0ce3-4a77-9e49-a40503d88657,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:01:15.747159,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54387,7 +54387,7 @@ class Problem { } }",247.23390000144718,,Completions generated successfully. Ready for display in your IDE.,61a7de69-7223-4fda-a709-b92802b32c99,1,deepseek-ai/deepseek-coder-1.3b-base," } else if (currentGrade > 3.3f) { -",216,1.0 +",216,1.0 2025-09-22T14:01:15.937279,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54419,7 +54419,7 @@ class Problem { } }",189.53329999931157,,Completions generated successfully. Ready for display in your IDE.,01203831-649b-43af-8077-a813d2d5870a,1,deepseek-ai/deepseek-coder-1.3b-base," ArrayList out = new ArrayList<>(); -",164,0.9703 +",164,0.9703 2025-09-22T14:01:16.069768,[1],"//In this Kata, you have to sort an array of non-negative integers according to // number of ones in their binary representation in ascending order. // For similar number of ones, sort based on decimal value. @@ -54440,7 +54440,7 @@ function sort_array(arr){ }); }",132.41829999606125,,Completions generated successfully. Ready for display in your IDE.,65ed7c63-9721-41e5-8f0a-1cecce8bbf5b,1,deepseek-ai/deepseek-coder-1.3b-base," return a - b; -",103,1.0 +",103,1.0 2025-09-22T14:01:16.289054,[1],"//It is the last week of the semester and the teacher has to give the grades // to students. The teacher has been making her own algorithm for grading. // The only problem is, she has lost the code she used for grading. @@ -54507,7 +54507,7 @@ function numerical_letter_grade(grades){ return grades_array; }",219.07659999851603,,Completions generated successfully. Ready for display in your IDE.,676b1312-a0e5-4540-8e0f-6a8ef8d3466b,1,deepseek-ai/deepseek-coder-1.3b-base," else if (grades[i] > 0.0){ -",190,1.0 +",190,1.0 2025-09-22T14:01:16.643993,[1],"//Write a function that takes a message, and encodes in such a // way that it swaps case of all letters, replaces all vowels in // the message with the letter that appears 2 places ahead of that @@ -54532,7 +54532,7 @@ function encode(message){ // not a vowel so just add to string without changing casing or anything result += letter; } -",329,0.8939 +",329,0.8939 2025-09-22T14:01:16.696322,[1],"from typing import List def f(n: int) -> List[int]: @@ -54553,7 +54553,7 @@ def f(n: int) -> List[int]: else: a.append(sum(range(1,i+1))) return a - pass",52.21030000393512,,Completions generated successfully. Ready for display in your IDE.,b0100fec-ff88-4482-ba87-73a539e9bcbe,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 + pass",52.21030000393512,,Completions generated successfully. Ready for display in your IDE.,b0100fec-ff88-4482-ba87-73a539e9bcbe,1,deepseek-ai/deepseek-coder-1.3b-base,,21,1.0 2025-09-22T14:01:16.805654,[1],"//From an array of integers, remove all elements that occur more than once. // Keep order of elements left the same as in the input. // >>> remove_duplicates([1, 2, 3, 2, 4]) @@ -54567,7 +54567,7 @@ function remove_duplicates(numbers){ return result; } ",108.93360000045504,,Completions generated successfully. Ready for display in your IDE.,63112131-247b-4258-8080-faa3f40d3bf8,1,deepseek-ai/deepseek-coder-1.3b-base," var result = []; -",83,1.0 +",83,1.0 2025-09-22T14:01:16.871014,[1],"//You are given an array arr of integers and you need to return // sum of magnitudes of integers multiplied by product of all signs // of each number in the array, represented by 1, -1 or 0. @@ -54599,7 +54599,7 @@ function prod_signs(arr){ } } ",65.20050000108313,,Completions generated successfully. Ready for display in your IDE.,c7ff4e04-026e-4da7-ae7a-1b1dab544948,1,deepseek-ai/deepseek-coder-1.3b-base," - ",42,1.0 + ",42,1.0 2025-09-22T14:01:16.990013,[1],"def encrypt(s: str) -> str: """"""Create a function encrypt that takes a string as an argument and returns a string encrypted with the alphabet being rotated. @@ -54619,7 +54619,7 @@ function prod_signs(arr){ output_string = """" for i in s: output_string += input_string[(input_string.index(i)+4)%26]",,118.78470000374364,,Completions generated successfully. Ready for display in your IDE.,622872b6-6bbc-4814-9304-017595651c2e,1,deepseek-ai/deepseek-coder-1.3b-base," - return output_string",93,0.9783 + return output_string",93,0.9783 2025-09-22T14:01:17.542079,[1],"from typing import Dict def check_dict_case(dict: Dict[str, str]) -> bool: @@ -54647,7 +54647,7 @@ def check_dict_case(dict: Dict[str, str]) -> bool: return True ",552.1311999982572,,Completions generated successfully. Ready for display in your IDE.,2292ec6e-f0c5-42d3-9ae2-e1cfc9e9bcee,1,deepseek-ai/deepseek-coder-1.3b-base," if not set(dict).issubset(set('')): -",517,0.9872 +",517,0.9872 2025-09-22T14:01:17.701443,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54698,7 +54698,7 @@ class Problem { } }",158.9955999952508,,Completions generated successfully. Ready for display in your IDE.,3a6e8430-4489-45e2-a1de-3a7499140325,1,deepseek-ai/deepseek-coder-1.3b-base," String extension = tokens[1]; -",130,1.0 +",130,1.0 2025-09-22T14:01:17.797840,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54730,7 +54730,7 @@ class Problem { } }",95.43229999690084,,Completions generated successfully. Ready for display in your IDE.,be587301-82c7-4c3d-acb4-1df87e1d0a81,1,deepseek-ai/deepseek-coder-1.3b-base," } -",68,1.0 +",68,1.0 2025-09-22T14:01:17.873738,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54766,7 +54766,7 @@ class Problem { } return (lst1.stream().allMatch(x -> x % 2 == 0)) ? ""YES"" : ""NO""; }",,76.02820000465726,,Completions generated successfully. Ready for display in your IDE.,db7b3b87-f94e-4474-8bae-1aaea9241737,1,deepseek-ai/deepseek-coder-1.3b-base," -}",49,1.0 +}",49,1.0 2025-09-22T14:01:17.960515,[1],"from typing import List def specialFilter(nums: List[int]) -> int: @@ -54788,7 +54788,7 @@ def specialFilter(nums: List[int]) -> int: return count ",86.77629999874625,,Completions generated successfully. Ready for display in your IDE.,c03cc0f1-dd53-4bbf-8503-f6f2bce86ed1,1,deepseek-ai/deepseek-coder-1.3b-base," continue -",64,1.0 +",64,1.0 2025-09-22T14:01:18.035057,[1],"//Given two arrays operator, and operand. The first array has basic algebra operations, and // the second array is an array of integers. Use the two given arrays to build the algebric // expression and return the evaluation of this expression. @@ -54855,7 +54855,7 @@ function do_algebra(operator, operand){ return answer; } ",73.77830000041286,,Completions generated successfully. Ready for display in your IDE.,a2fcc115-2116-45de-ac2d-9c29d774a2a0,1,deepseek-ai/deepseek-coder-1.3b-base," -",41,1.0 +",41,1.0 2025-09-22T14:01:18.161881,[1],"//Circular shift the digits of the integer x, shift the digits right by shift // and return the result as a string. // If shift > number of digits, return digits reversed. @@ -54876,7 +54876,7 @@ function circular_shift(x, shift){ } ",127.08540000312496,,Completions generated successfully. Ready for display in your IDE.,c11547d8-ce78-4f2f-93e8-eb2e92bf8bc5,1,deepseek-ai/deepseek-coder-1.3b-base," if (shift >= len){ -",103,0.9438 +",103,0.9438 2025-09-22T14:01:18.477905,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54920,7 +54920,7 @@ class Problem { } }",315.8402000044589,,Completions generated successfully. Ready for display in your IDE.,1e5c4088-eae7-4009-bd9c-95d348971406,1,deepseek-ai/deepseek-coder-1.3b-base," return nums.stream().filter(x -> x > 10).filter(x -> { -",288,1.0 +",288,1.0 2025-09-22T14:01:18.757226,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -54985,7 +54985,7 @@ class Problem { } }",278.8460000010673,,Completions generated successfully. Ready for display in your IDE.,ccdf22f4-4e2e-4e61-b387-ade16a177025,1,deepseek-ai/deepseek-coder-1.3b-base," for (int i = 0; i < grades.size(); ++i) { -",248,1.0 +",248,1.0 2025-09-22T14:01:19.454565,[1],"//Given an array of strings, where each string consists of only digits, return an array. // Each element i of the output should be ""the number of odd elements in the // string i of the input."" where all the i's should be replaced by the number @@ -55010,7 +55010,7 @@ function odd_count(lst){ // console.log(""The sum is: "", odd_sum) var odd_word = 'the number of odd elements ' + odd_sum + ' n the st'+ String(i+1).padStart(2,'r')+ - 'ring ' + odd",673,0.9015 + 'ring ' + odd",673,0.9015 2025-09-22T14:01:19.517911,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -55053,7 +55053,7 @@ class Problem { }).count(); } }",62.973100000817794,,Completions generated successfully. Ready for display in your IDE.,15a51906-150f-490c-8890-974595ee0323,1,deepseek-ai/deepseek-coder-1.3b-base," -",38,1.0 +",38,1.0 2025-09-22T14:01:19.653105,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -55095,7 +55095,7 @@ class Problem { if (c1 == 0 || c2 == 0) return ""Yes"";"," } }",135.2365999991889,,Completions generated successfully. Ready for display in your IDE.,66ef6456-217c-4c14-b68e-c8a2a5088c2a,1,deepseek-ai/deepseek-coder-1.3b-base," else return ""No""; -",110,1.0 +",110,1.0 2025-09-22T14:01:19.800773,[1],"from typing import Union def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[int, float, str, None]: @@ -55172,7 +55172,7 @@ def compare_one(a: Union[int, float, str], b: Union[int, float, str]) -> Union[i elif float(a_tmp) < b: return b",147.25569999427535,,Completions generated successfully. Ready for display in your IDE.,91c3f032-ee3a-4ccf-8dce-f3dc031d78df,1,deepseek-ai/deepseek-coder-1.3b-base," elif a < b: -",118,1.0 +",118,1.0 2025-09-22T14:01:19.990046,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -55212,7 +55212,7 @@ class Problem { } }",189.11250000383006,,Completions generated successfully. Ready for display in your IDE.,6e98ee83-8069-4f8e-815b-03167e514379,1,deepseek-ai/deepseek-coder-1.3b-base," return o1.compareTo(o2); -",161,1.0 +",161,1.0 2025-09-22T14:01:20.215083,[1],"//Find the shortest palindrome that begins with a supplied string. // Algorithm idea is simple: // - Find the longest postfix of supplied string that is a palindrome. @@ -55238,7 +55238,7 @@ function make_palindrome(string){ } ",224.65650000231108,,Completions generated successfully. Ready for display in your IDE.,55928360-644b-4795-994c-f4a974492cfb,1,deepseek-ai/deepseek-coder-1.3b-base," //find length of string var string_length = string.length; -",186,0.9462 +",186,0.9462 2025-09-22T14:01:20.343542,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -55274,7 +55274,7 @@ class Problem { } }",128.3391000033589,,Completions generated successfully. Ready for display in your IDE.,eb60d554-f56b-4faf-80d7-fa717f797a1a,1,deepseek-ai/deepseek-coder-1.3b-base," max = val; -",100,1.0 +",100,1.0 2025-09-22T14:01:20.452390,[1],"import java.util.*; import java.lang.reflect.*; import org.javatuples.*; @@ -55298,4 +55298,4 @@ class Problem { }"," } }",108.67159999907017,,Completions generated successfully. Ready for display in your IDE.,469a9cd6-3163-4746-be40-e9a07caacc4a,1,deepseek-ai/deepseek-coder-1.3b-base," return result; -",84,1.0 +",84,1.0 diff --git a/src/performance_and_analysis/chat_benchmark_results.csv b/src/performance_and_analysis/chat_benchmark_results.csv index 409ed6a..2e378a8 100644 --- a/src/performance_and_analysis/chat_benchmark_results.csv +++ b/src/performance_and_analysis/chat_benchmark_results.csv @@ -1,4 +1,4 @@ -timestamp,model_ids,user_prompt,e2e_time_ms,error,chat_id,response_title,assistant_completion,generation_time_ms,confidence +timestamp,model_ids,user_prompt,e2e_time_ms,error,chat_id,response_title,assistant_completion,generation_time_ms,confidence 2025-09-22T22:26:11.629984,[3],Please explain the different components of the solution for the following problem: Write a python function to remove first and last occurrence of a given character from the string.,8808.68640000699,,60ca99b0-e605-4353-b786-6e00c99f0142,Remove first and last occurrence of a given character from a string," The solution involves creating a function that takes three parameters: the input string, the character to be removed, and an optional parameter indicating whether to remove only the first or last occurrence. The function will then iterate through the string and check if the character matches the target character. If it does, the function will either remove the first or last occurrence based on the optional parameter. @@ -14,7 +14,7 @@ Here's a step-by-step breakdown of the solution: Here's the implementation of the function: ```python -def remove_occurrences(string, char, first_last",8765,0.0 +def remove_occurrences(string, char, first_last",8765,0.0 2025-09-22T22:26:20.424422,[3],Please explain the different components of the solution for the following problem: Write a function to sort a given matrix in ascending order according to the sum of its rows.,8794.24740000104,,008a15b0-11b3-47ce-96c6-d46d13bf312d,Sorting Matrix by Row Sums," To solve this problem, we need to follow these steps: @@ -51,7 +51,7 @@ print(sorted_matrix) This will output: ``` -[[4, 5], [7,",8765,0.0 +[[4, 5], [7,",8765,0.0 2025-09-22T22:26:29.212510,[3],Please explain the different components of the solution for the following problem: Write a function to count the most common words in a dictionary.,8787.216400000034,,6d748fdd-d630-427b-9f16-92989fcef0bf,Components of Solution," The solution involves several steps: @@ -82,7 +82,7 @@ def count_most_common_words(text): # Step 4: Find the most common word(s) max_frequency = max(word_count.values()) - most_common_words = [word for word, freq in",8761,0.0 + most_common_words = [word for word, freq in",8761,0.0 2025-09-22T22:26:38.060337,[3],Please explain the different components of the solution for the following problem: Write a python function to find the volume of a triangular prism.,8848.084400000516,,1024d15e-2b04-41d1-902f-e5d8ff648f11,Components of Volume Calculation Function," The volume V of a triangular prism can be calculated using the formula: @@ -115,7 +115,7 @@ def calculate_volume(base_area, height): # Example usage base_area = 50 # Example base area -height",8762,0.0 +height",8762,0.0 2025-09-22T22:26:44.096993,[3],Please explain the different components of the solution for the following problem: Write a function to split a string at lowercase letters.,6035.633899999084,,cf49956b-3b61-4bda-b2e0-45c03d9f4068,String splitting by lowercase letters," The solution involves creating a function that iterates through each character in the string and checks if it is a lowercase letter. If it is, the function splits the string at that point. @@ -130,7 +130,7 @@ def split_at_lowercase(s): print(split_at_lowercase(""HelloWorld"")) # Output: ['H', 'e', 'l', 'l', 'oW', 'o', 'r', 'l', 'd'] ``` -This function works by first creating a new string containing only non-lowercase characters from the input string `s`. It then uses this new string as a delimiter to split the original string into a list of substrings.",6008,0.0 +This function works by first creating a new string containing only non-lowercase characters from the input string `s`. It then uses this new string as a delimiter to split the original string into a list of substrings.",6008,0.0 2025-09-22T22:26:52.885290,[3],Please explain the different components of the solution for the following problem: Write a function to find sequences of lowercase letters joined with an underscore.,8788.721399992937,,81afabc6-7022-455a-8a22-215b0d9b9c30,Components of sequence finding function," The solution involves creating a function that can identify sequences of lowercase letters joined by underscores. Here are the key components: @@ -169,7 +169,7 @@ Putting it all together: import re def find_sequences(input_string): - pattern = r'[a-z]+_' # Regex pattern to",8761,0.0 + pattern = r'[a-z]+_' # Regex pattern to",8761,0.0 2025-09-22T22:26:58.818521,[3],Please explain the different components of the solution for the following problem: Write a function to find the perimeter of a square.,5932.357300000149,,6bbf6d1b-6706-4bfd-b19a-ef5b56bca6f5,Components of Square Perimeter Function," To solve this problem, we need to create a function that calculates the perimeter of a square given its side length. Here are the main components of the solution: @@ -188,7 +188,7 @@ def calculate_perimeter(side_length): return perimeter ``` -This function can then be used by calling it with the appropriate side length as an argument.",5904,0.0 +This function can then be used by calling it with the appropriate side length as an argument.",5904,0.0 2025-09-22T22:27:05.822433,[3],Please explain the different components of the solution for the following problem: Write a function to remove characters from the first string which are present in the second string.,7004.094399992027,,c547acdf-c954-4954-9cc2-95fd50718b98,Function to Remove Characters," The solution involves creating a function that takes two strings as input. The function will iterate through each character in the first string and check if it exists in the second string. If the character does not exist in the second string, it will be added to a new string. Finally, the function will return this new string. @@ -209,7 +209,7 @@ str2 = ""world"" print(remove_chars(str1, str2)) # Output: ""helo"" ``` -This function works by iterating over each character in `str1` and checking if it is not in `str2`. If it is not found in `str2`, it is appended to the `result` string.",6971,0.0 +This function works by iterating over each character in `str1` and checking if it is not in `str2`. If it is not found in `str2`, it is appended to the `result` string.",6971,0.0 2025-09-22T22:27:14.518648,[3],Please explain the different components of the solution for the following problem: Write a function to find whether a given array of integers contains any duplicate element.,8695.679800002836,,c6222eb0-ecb5-46c6-9ca0-94339309152e,Explanation of Solution Components," The solution can be implemented using various approaches. Here are some key components that could be part of the solution: @@ -222,7 +222,7 @@ The solution can be implemented using various approaches. Here are some key comp 4. **Brute Force**: Iterating through each element and checking against all other elements. This method has a time complexity of O(n^2) and a space complexity of O(1). -Each component has its own advantages and trade-offs based on the specific requirements of the problem, such as the size of the input array and available memory.",8659,0.0 +Each component has its own advantages and trade-offs based on the specific requirements of the problem, such as the size of the input array and available memory.",8659,0.0 2025-09-22T22:27:23.306783,[3],Please explain the different components of the solution for the following problem: Write a function to check if the given number is woodball or not.,8788.101400001324,,942c2233-b670-4404-b6a5-aced841c89d8,Woodball Check Function Components," The problem requires us to create a function that checks whether a given number is a ""woodball"" or not. A ""woodball"" can be defined as a number that has at least one digit that is divisible by 5. @@ -251,7 +251,7 @@ def is_woodball(number): #### 2. Digit Extraction To extract each digit from the number, we can convert the number to a string and then iterate over its characters. -```",8761,0.0 +```",8761,0.0 2025-09-22T22:27:32.095961,[3],Please explain the different components of the solution for the following problem: Write a function to find m number of multiples of n.,8788.61859999597,,88968076-413a-4f0e-8501-4a334f1a85db,Components of Solution," To solve this problem, we need to create a function that generates `m` multiples of `n`. Here are the key components of the solution: @@ -276,7 +276,7 @@ def find_multiples(n, m): print(find_multiples(3, 5)) # Output: [3, 6, 9, 12, 15] ``` -This function will generate and return a list containing the first `m` multiples of `",8759,0.0 +This function will generate and return a list containing the first `m` multiples of `",8759,0.0 2025-09-22T22:27:40.885350,[3],Please explain the different components of the solution for the following problem: Write a function to find the first duplicate element in a given array of integers.,8789.134299993748,,b3a9a338-fad1-4201-a86f-fbe4e7065c43,Components of Solution," To solve this problem, we need to create a function that can efficiently find the first duplicate element in an array of integers. Here are the main components of the solution: @@ -305,7 +305,7 @@ seen = {} ``` ### Step 3: Iteration -Traverse through the array and check if each element is in the `seen` dictionary",8762,0.0 +Traverse through the array and check if each element is in the `seen` dictionary",8762,0.0 2025-09-22T22:27:49.696782,[3],Please explain the different components of the solution for the following problem: Write a python function to find the maximum sum of elements of list in a list of lists.,8811.211300009745,,5392571b-5012-49a7-81d3-77a4d9eb5a73,Components of Solution," To solve this problem, we need to create a Python function that can iterate through each sublist in a list of lists and calculate the sum of its elements. The function should then return the maximum sum found among all the sublists. @@ -331,7 +331,7 @@ def max_sum_of_sublists(lst): # Update the maximum sum if the current sum is greater if current_sum > max_sum: - max_sum = current_sum",8764,0.0 + max_sum = current_sum",8764,0.0 2025-09-22T22:27:58.527944,[3],Please explain the different components of the solution for the following problem: Write a function to convert the given binary number to its decimal equivalent.,8830.868400007603,,e5a652b7-bf34-46c5-83c2-f185aa61a1be,Binary to Decimal Conversion," The different components of the solution for converting a binary number to its decimal equivalent include: @@ -364,7 +364,7 @@ binary_number = ""1101"" print(f""The decimal equivalent of {binary_number} is {binary_to_decimal(binary_number)}"") ``` -This function takes a",8760,0.0 +This function takes a",8760,0.0 2025-09-22T22:28:07.321105,[3],Please explain the different components of the solution for the following problem: Write a python function to find the product of non-repeated elements in a given array.,8793.208600007347,,8f62e1f6-1a48-42cb-ae12-2ff851429b8d,Components of Solution," To solve this problem, we need to identify the unique (non-repeated) elements in the given array and then compute their product. @@ -396,7 +396,7 @@ Using the counts obtained from step 1, filter out elements that have a count of ```python def get_non_repeated_elements(element_count): - return [key for key",8765,0.0 + return [key for key",8765,0.0 2025-09-22T22:28:16.110599,[3],Please explain the different components of the solution for the following problem: Write a function to check if the given tuple list has all k elements.,8788.770800005295,,178bc4b3-feb5-4a9f-b048-37ad5e6fdacc,Tuple List Checker," The solution involves creating a function that iterates through each tuple in the list and checks if it contains exactly k elements. Here are the steps: @@ -427,7 +427,7 @@ print(check_tuple_list([(1, 2), (3, 4, 5), (6)], 2)) # Output: [True, False, False] ``` -This will print `[True",8761,0.0 +This will print `[True",8761,0.0 2025-09-22T22:28:24.899060,[3],Please explain the different components of the solution for the following problem: Write a python function to remove all digits from a list of strings.,8787.727999995695,,5c67795a-e562-44f6-8cf5-d776279694bb,Remove Digits from List of Strings," To solve this problem, we need to create a Python function that takes a list of strings and removes all digits from each string in the list. @@ -452,7 +452,7 @@ def remove_digits(strings): return result # Example usage -strings = [""hello123"", ""world456"", ""test789",8760,0.0 +strings = [""hello123"", ""world456"", ""test789",8760,0.0 2025-09-22T22:28:31.961089,[3],Please explain the different components of the solution for the following problem: Write a python function to find binomial co-efficient.,7061.871699988842,,7541d35e-aeeb-4f15-bd31-499fe7d18246,Binomial Coefficient Calculation," The binomial coefficient (also known as ""n choose k"") can be calculated using various methods. Here are some common approaches: @@ -467,7 +467,7 @@ The binomial coefficient (also known as ""n choose k"") can be calculated using 5. **Using Libraries**: Some libraries like NumPy or SciPy offer built-in functions to compute the binomial coefficient efficiently. -Each of these methods has its own advantages and trade-offs depending on the specific requirements and constraints of your application.",7036,0.0 +Each of these methods has its own advantages and trade-offs depending on the specific requirements and constraints of your application.",7036,0.0 2025-09-22T22:28:40.745944,[3],Please explain the different components of the solution for the following problem: Write a python function to find the element occurring odd number of times.,8785.22749998956,,6386609a-ae8d-4d00-a746-b7dfc20419d1,Odd Occurrences Element Finder," The solution involves creating a function that takes a list of elements as input and returns the element that occurs an odd number of times. Here are the steps to achieve this: @@ -499,7 +499,7 @@ def find_odd_occurrence(elements): if value % 2!= 0: return key - #",8760,0.0 + #",8760,0.0 2025-09-22T22:28:49.536590,[3],Please explain the different components of the solution for the following problem: Write a python function to count all the substrings starting and ending with same characters.,8790.105299995048,,8fd9aed0-fa4a-43db-af34-288ada9f6bf6,Substring counting function explanation," The solution involves creating a function that iterates through each character in the string and checks if there is a substring starting and ending with the same character. The function will then increment a counter for every valid substring found. @@ -528,7 +528,7 @@ In this function: - For each substring, we check if its first and last characters are the same. - If they are, we increment the `count`. -This approach ensures that all possible substrings are considered, and the function returns the total count of those that meet the criteria",8764,0.0 +This approach ensures that all possible substrings are considered, and the function returns the total count of those that meet the criteria",8764,0.0 2025-09-22T22:28:58.334995,[3],Please explain the different components of the solution for the following problem: Write a function to find the top k integers that occur most frequently from given lists of sorted and distinct integers using heap queue algorithm.,8797.53019999771,,fa45ffab-1ceb-48e1-ad0d-65260ca1ced2,Heap Queue Algorithm for Top K Integers," To solve this problem, we can utilize the heap queue (priority queue) data structure provided by Python's `heapq` module. Here’s how we can break down the solution into its components: @@ -554,7 +554,7 @@ def find_top_k_frequent_integers(lists): min_heap = [] # Step 3: Frequency Counting - for num in combined",8769,0.0 + for num in combined",8769,0.0 2025-09-22T22:29:07.125139,[3],Please explain the different components of the solution for the following problem: Write a python function to find the largest prime factor of a given number.,8789.93330000958,,b758ec42-b063-4b76-b327-34dbef9439c2,Prime Factorization Function," The solution involves creating a function that takes an integer input and returns its largest prime factor. Here’s how we can break down the steps: @@ -582,7 +582,7 @@ def largest_prime_factor(n): n //= 2 # Check for odd factors from 3 upwards - factor",8760,0.0 + factor",8760,0.0 2025-09-22T22:29:15.674357,[3],Please explain the different components of the solution for the following problem: Write a python function to convert a decimal number to binary number.,8549.057600001106,,422c014c-b062-4ce5-b86a-7d201c80eeee,Convert Decimal to Binary," The different components of the solution include: @@ -606,7 +606,7 @@ def decimal_to_binary(decimal_number): return binary_representation ``` -This function first checks if the input is valid (non-negative integer). Then, it uses a loop to convert the decimal number to binary by repeatedly dividing the number by 2 and appending the remainder to the result string. Finally, it returns the binary representation as a string.",8517,0.0 +This function first checks if the input is valid (non-negative integer). Then, it uses a loop to convert the decimal number to binary by repeatedly dividing the number by 2 and appending the remainder to the result string. Finally, it returns the binary representation as a string.",8517,0.0 2025-09-22T22:29:24.462130,[3],Please explain the different components of the solution for the following problem: Write a python function to find the missing number in a sorted array.,8787.804300009157,,938b5eab-08de-42c8-a3c4-12407becc2ec,Missing Number Solution Components," The solution involves creating a function that takes an input list of integers and returns the missing number. Here are the main steps: @@ -632,7 +632,7 @@ def find_missing_number(arr): # Step 3: Actual sum calculation actual_sum = sum(arr) - # Step 4: Missing",8760,0.0 + # Step 4: Missing",8760,0.0 2025-09-22T22:29:33.248467,[3],Please explain the different components of the solution for the following problem: Write a function to find the n-th rectangular number.,8786.096600000747,,93ad93af-d8e7-471a-b1f4-f1bd1b7c1f75,Explanation of Rectangular Number Function," The rectangular numbers are those that can be expressed as the product of two integers. The n-th rectangular number is the nth number in this sequence. @@ -659,7 +659,7 @@ def nth_rectangular_number(n): while len(rectangular_numbers) < n: j = i + 1 while j <= i: - product = i * j",8757,0.0 + product = i * j",8757,0.0 2025-09-22T22:29:42.043964,[3],Please explain the different components of the solution for the following problem: Write a python function to find the nth digit in the proper fraction of two given numbers.,8794.595900006243,,dc10335b-cc45-4a3f-b4c2-d9af9d4630bb,Components of Solution," The solution involves creating a function that takes two integers as input and returns the nth digit of the proper fraction formed by dividing these two numbers. @@ -694,7 +694,7 @@ Combining all these steps into one function: ```python def find_nth_digit_in_proper_fraction(num1, num2, n): fraction = calculate_proper_fraction(num1, num2) - string_rep = convert_to",8765,0.0 + string_rep = convert_to",8765,0.0 2025-09-22T22:29:50.835859,[3],Please explain the different components of the solution for the following problem: Write a function to sort a given mixed list of integers and strings.,8792.172000001301,,2c50f42e-3458-4e1d-9bd0-9501e1d205d8,Components of sorting function," The solution involves creating a custom sorting function that can handle both integers and strings. Here are the main components: @@ -726,7 +726,7 @@ def custom_sort(mixed_list): return sorted(mixed_list, key=cmp_to_key(compare)) # Example usage: -mixed_list = [3, 'apple', 1, 'banana', 2, 'cherry']",8762,0.0 +mixed_list = [3, 'apple', 1, 'banana', 2, 'cherry']",8762,0.0 2025-09-22T22:29:59.627569,[3],Please explain the different components of the solution for the following problem: Write a function to find the division of first even and odd number of a given list.,8791.076499997871,,70b6292c-0860-415b-bb43-c75894ae3a22,Components of Solution," The solution involves several steps: @@ -763,7 +763,7 @@ def find_division(lst): # Example usage numbers = [1, 2, 3, 4, 5, 6] -result",8763,0.0 +result",8763,0.0 2025-09-22T22:30:08.428613,[3],Please explain the different components of the solution for the following problem: Write a function to check if the letters of a given string can be rearranged so that two characters that are adjacent to each other are different.,8801.014199998463,,f064df76-d908-439d-98d3-2c055639fa58,Explanation of Solution Components," The solution involves several steps: @@ -798,7 +798,7 @@ print(can_rearrange(""aabbcc"")) # Output: False print(can_rearrange(""abcde"")) # Output: True ``` -This function",8774,0.0 +This function",8774,0.0 2025-09-22T22:30:17.219699,[3],Please explain the different components of the solution for the following problem: Write a function to find frequency of the elements in a given list of lists using collections module.,8791.024099991773,,ef25af39-f834-41d7-bad7-486b7b781daf,Components of Solution," To solve this problem, we need to break down the task into several steps: @@ -832,7 +832,7 @@ nested_list = [[1, 2, 2], [3, 3, 3, 4], [4, 4]] print(find_frequencies(nested_list)) ``` -This function will correctly compute the frequency of each",8764,0.0 +This function will correctly compute the frequency of each",8764,0.0 2025-09-22T22:30:25.110393,[3],Please explain the different components of the solution for the following problem: Write a function to filter even numbers using lambda function.,7890.150899998844,,2749d958-880e-48fe-9458-cdc3dcf39210,Filtering Even Numbers with Lambda Function," The solution involves creating a list comprehension or a map function that uses a lambda function to check if each number is even. Here are the steps: @@ -856,7 +856,7 @@ even_numbers = [x for x in numbers if x % 2 == 0] print(even_numbers) ``` -Both methods will give you a list of even numbers from the original list.",7863,0.0 +Both methods will give you a list of even numbers from the original list.",7863,0.0 2025-09-22T22:30:33.898758,[3],Please explain the different components of the solution for the following problem: Write a python function to find the sum of repeated elements in a given array.,8787.998699990567,,bd690403-9750-4999-9728-c8e10d0b70b3,Components of Solution," To solve this problem, we need to create a Python function that finds the sum of repeated elements in an array. Here are the main components of the solution: @@ -889,7 +889,7 @@ def sum_of_repeated_elements(arr): return total_sum # Example usage: -arr =",8763,0.0 +arr =",8763,0.0 2025-09-22T22:30:42.697590,[3],Please explain the different components of the solution for the following problem: Write a function to find sequences of lowercase letters joined with an underscore using regex.,8798.702400003094,,5768e151-7a54-4e3c-87aa-88dbea6be9f1,Regex Solution for Underscore Joined Letters," The solution involves creating a regular expression pattern that matches sequences of lowercase letters followed by underscores. Here’s how you can do it step-by-step: @@ -916,7 +916,7 @@ input_str = ""hello_world_this_is_a_test"" print(find_underscore_joined_sequences(input_str)) ``` -In this example, the function `find_underscore_joined_sequences` takes an input string and returns a list of all sequences of lowercase letters joined by an underscore. The regex pattern `[a-z]+_` ensures that only sequences of one or more lowercase letters followed",8762,0.0 +In this example, the function `find_underscore_joined_sequences` takes an input string and returns a list of all sequences of lowercase letters joined by an underscore. The regex pattern `[a-z]+_` ensures that only sequences of one or more lowercase letters followed",8762,0.0 2025-09-22T22:30:51.483783,[3],Please explain the different components of the solution for the following problem: Write a function that matches a word at the beginning of a string.,8785.730800009333,,38f580d6-3488-4e0d-9987-d44998ca50b8,Components of Solution," To solve this problem, we need to break down the task into several components: @@ -943,7 +943,7 @@ print(starts_with_word(""hello world"", ""hello"")) # Output: True print(starts_with_word(""hello world"", ""world"")) # Output: False ``` -This function works by checking if the length of the word is greater than the length of the main string (in which case it cannot be a valid match), then comparing the first `len(word",8759,0.0 +This function works by checking if the length of the word is greater than the length of the main string (in which case it cannot be a valid match), then comparing the first `len(word",8759,0.0 2025-09-22T22:31:00.277594,[3],Please explain the different components of the solution for the following problem: Write a function to find the gcd of the given array elements.,8793.948800012004,,26b40c05-e56d-48b0-9d4b-72ca7ee35fae,GCD Function Components," The GCD (Greatest Common Divisor) of an array can be found by iteratively applying the Euclidean algorithm to pairs of numbers. Here are the key components of the solution: @@ -960,7 +960,7 @@ Here's a step-by-step breakdown of how these components work together: 1. Define the helper function `gcd(a, b)` which computes the GCD of two integers using the Euclidean algorithm. 2. Implement the main function `array_gcd(arr)` which uses the helper function to compute the GCD of the array elements. -3. Iterate through the array, updating the",8763,0.0 +3. Iterate through the array, updating the",8763,0.0 2025-09-22T22:31:05.499848,[3],Please explain the different components of the solution for the following problem: Write a python function to determine whether all the numbers are different from each other are not.,5221.558200006257,,c6c013a7-3109-496c-b364-8bafba4da48e,Components of Solution," The solution involves creating a Python function that checks if all elements in an input list are unique. Here are the main components: @@ -977,7 +977,7 @@ def check_unique_numbers(numbers): return len(set(numbers)) == len(numbers) ``` -This function takes a list of numbers, converts it to a set, and compares their lengths to determine uniqueness.",5189,0.0 +This function takes a list of numbers, converts it to a set, and compares their lengths to determine uniqueness.",5189,0.0 2025-09-22T22:31:14.288740,[3],Please explain the different components of the solution for the following problem: Write a python function to find the last digit when factorial of a divides factorial of b.,8789.274100010516,,206ec519-499d-4dd9-b67c-d2a87be501cf,Factorial Division Last Digit," The solution involves several steps: @@ -1011,7 +1011,7 @@ def last_digit_fact_division(a, b): return last_digit # Example usage -print(last_digit_fact_division(5, 7)) # Output will be the last digit of 7! / 5",8763,0.0 +print(last_digit_fact_division(5, 7)) # Output will be the last digit of 7! / 5",8763,0.0 2025-09-22T22:31:23.076174,[3],Please explain the different components of the solution for the following problem: Write a python function to set all odd bits of a given number.,8786.091000001761,,064d2f68-64f9-4df2-beff-45fd87b6a445,Explanation of Odd Bit Setting Function," The solution involves creating a function that takes an integer input and sets all its odd bits to 1. Here's how we can do it step by step: @@ -1026,7 +1026,7 @@ The solution involves creating a function that takes an integer input and sets a - For example, to set bits at positions 0, 2, 4,..., 30, we would use `mask = 1 | (1 << 2) | (1 << 4) |...`. 4. **Applying the Mask**: - Apply the mask to the input number using bitwise AND operation. -5. **Returning",8760,0.0 +5. **Returning",8760,0.0 2025-09-22T22:31:30.590521,[3],Please explain the different components of the solution for the following problem: Write a function to extract every first or specified element from a given two-dimensional list.,7514.1988999967,,af1dbe3b-69fa-4183-b527-b628c6edd033,Extract elements from 2D list," The solution can be implemented using nested loops or list comprehension. Here's an example using both methods: @@ -1049,7 +1049,7 @@ def extract_elements_using_list_comprehension(matrix, index): Both functions take a `matrix` (a two-dimensional list) and an `index` as input parameters. The `extract_elements_using_nested_loops` function uses nested loops to iterate through each row and column of the matrix, while the `extract_elements_using_list_comprehension` function uses list comprehension to achieve the same result more concisely. -You can choose either method based on your preference or specific requirements.",7486,0.0 +You can choose either method based on your preference or specific requirements.",7486,0.0 2025-09-22T22:31:39.381780,[3],Please explain the different components of the solution for the following problem: Write a function to find the list with minimum length using lambda function.,8791.582999998354,,3de828f2-e7c7-48a3-b292-8f024ab01cc2,Components of Solution," The solution involves creating a function that takes a list of lists as input and returns the list with the minimum length. This can be achieved by using the `min()` function along with a lambda function to specify the key for comparison. @@ -1074,7 +1074,7 @@ Here’s how you can implement this: def find_min_length_list(lists): return min(lists, key=lambda x: len(x)) -# Example",8760,0.0 +# Example",8760,0.0 2025-09-22T22:31:48.173610,[3],Please explain the different components of the solution for the following problem: Write a function to print check if the triangle is equilateral or not.,8791.481500011287,,b8bb7dfe-bf47-48d3-853e-eee128706757,Triangle Check Function," The function should take three sides as input and return True if they form an equilateral triangle, False otherwise. @@ -1108,7 +1108,7 @@ def is_equilateral(a, b, c): # Example usage: print(is_equilateral(3, 3, 3)) # Output: True -print(is_",8762,0.0 +print(is_",8762,0.0 2025-09-22T22:31:54.108667,[3],Please explain the different components of the solution for the following problem: Write a function to caluclate area of a parallelogram.,5934.686499997042,,36930c46-d5fa-4040-94ee-433bc64c43d8,Components of Parallelogram Area Calculation Function," The different components of the solution for calculating the area of a parallelogram include: @@ -1126,7 +1126,7 @@ def calculate_parallelogram_area(base, height): return base * height ``` -This function will correctly compute the area of a parallelogram given valid inputs.",5907,0.0 +This function will correctly compute the area of a parallelogram given valid inputs.",5907,0.0 2025-09-22T22:32:02.660510,[3],Please explain the different components of the solution for the following problem: Write a python function to check whether the first and last characters of a given string are equal or not.,8551.682899997104,,33a0b5e3-9c61-4b78-8aee-2dbd5270212d,Components of Solution," The solution involves creating a Python function that takes a string as input and checks if its first and last characters are equal. Here are the main components: @@ -1156,7 +1156,7 @@ def check_first_last_char(input_string): return False ``` -This function will correctly determine if the first and last characters of a given string are equal.",8525,0.0 +This function will correctly determine if the first and last characters of a given string are equal.",8525,0.0 2025-09-22T22:32:11.449431,[3],Please explain the different components of the solution for the following problem: Write a function to sort the given array by using counting sort.,8788.623900007224,,0998167e-4566-47ee-a642-7b4f29e2cb2a,Counting Sort Components," Counting sort is an efficient algorithm for sorting integers within a specific range. Here are its key components: @@ -1186,7 +1186,7 @@ def counting_sort(arr): # Step 1: Initialize the frequency array freq = [0] * (max_val - min_val + 1) - # Step 2: Populate the frequency array",8761,0.0 + # Step 2: Populate the frequency array",8761,0.0 2025-09-22T22:32:19.792407,[3],Please explain the different components of the solution for the following problem: Write a function to find t-nth term of geometric series.,8342.65370000503,,cd3bc21c-c83f-44f7-9150-4b0bb5cdce6a,Geometric Series Function Components," The components of the solution for finding the n-th term of a geometric series include: @@ -1216,7 +1216,7 @@ def nth_term_geometric_series(a, r, n): return a * r**(n - 1) ``` -This function takes the initial term `a`, common ratio `r`, and the desired term position `n` as inputs and returns the n-th term using the formula provided above.",8311,0.0 +This function takes the initial term `a`, common ratio `r`, and the desired term position `n` as inputs and returns the n-th term using the formula provided above.",8311,0.0 2025-09-22T22:32:27.238385,[3],Please explain the different components of the solution for the following problem: Write a python function to check if a given number is one less than twice its reverse.,7445.844899993972,,e6776f5c-e90d-4bcc-ba99-52feb9632366,Explanation of Solution Components," The function should take an integer input and return True or False based on the following condition: @@ -1241,7 +1241,7 @@ def is_one_less_than_twice_reverse(n): return result ``` -This function will correctly determine whether the given number satisfies the specified condition.",7420,0.0 +This function will correctly determine whether the given number satisfies the specified condition.",7420,0.0 2025-09-22T22:32:35.064635,[3],Please explain the different components of the solution for the following problem: Write a python function to find the largest number that can be formed with the given digits.,7825.696899992181,,c8b12a44-bfe9-4fd5-b0cd-fc7e58a146c0,Components of Solution," The solution involves several steps: @@ -1269,7 +1269,7 @@ def largest_number(digits): print(largest_number(['3', '4', '5'])) # Output: ""543"" ``` -This function takes a list of digit strings, converts them to integers, sorts them in descending order, and then joins them back together to form the largest possible number.",7798,0.0 +This function takes a list of digit strings, converts them to integers, sorts them in descending order, and then joins them back together to form the largest possible number.",7798,0.0 2025-09-22T22:32:43.855965,[3],Please explain the different components of the solution for the following problem: Write a python function to check whether the given two integers have opposite sign or not.,8790.95700000471,,f70f04ea-a83d-47fc-8a4a-f15fe1b14d5d,Opposite Sign Check Function," To solve this problem, we need to create a Python function that checks if two given integers have opposite signs. Here are the steps to achieve this: @@ -1296,7 +1296,7 @@ def check_opposite_sign(num1, num2): # Test cases print(check_opposite_sign(5, -3)) # Output: True -print(check_opposite_sign(-4, 7)) # Output: True",8763,0.0 +print(check_opposite_sign(-4, 7)) # Output: True",8763,0.0 2025-09-22T22:32:52.641006,[3],Please explain the different components of the solution for the following problem: Write a function to find the nth octagonal number.,8784.65440000582,,714f3804-20fb-4982-b9ee-998e0a58f659,Octagonal Number Calculation," The formula for the nth octagonal number is given by: @@ -1325,7 +1325,7 @@ print(octagonal_number(2)) print(octagonal_number(3)) ``` -This will output the first three octagonal numbers respectively",8759,0.0 +This will output the first three octagonal numbers respectively",8759,0.0 2025-09-22T22:33:01.432144,[3],Please explain the different components of the solution for the following problem: Write a function to find the maximum length of the subsequence with difference between adjacent elements for the given array.,8790.779799994198,,62148ec2-88e2-49c4-a539-b8126460f538,Explanation of Solution Components," The solution involves several steps to solve the problem efficiently. Here’s an overview of the key components: @@ -1352,7 +1352,7 @@ def max_length_subsequence(arr): for i in range(1, len(arr)): if abs(arr[i] - arr[i - 1]) <= 1: - curr_len +=",8766,0.0 + curr_len +=",8766,0.0 2025-09-22T22:33:10.224064,[3],Please explain the different components of the solution for the following problem: Write a python function to count number of substrings with the sum of digits equal to their length.,8792.019500004244,,3d31d340-d04b-4bc7-8b9c-acba0a3193ba,Counting Substrings with Sum Equal to Length," To solve this problem, we need to break down the task into several steps: @@ -1387,7 +1387,7 @@ def count_substrings_with_equal_length_and_digit_sum(s): # Example usage input_string = ""abc12"" result = count_substrings_with_equal_length_and_digit_sum(input_string) -print(f""Number of valid",8765,0.0 +print(f""Number of valid",8765,0.0 2025-09-22T22:33:16.771677,[3],Please explain the different components of the solution for the following problem: Write a python function to find smallest number in a list.,6547.512299992377,,b60f570e-ea51-48d2-9ff2-d425e44c99db,Components of Solution," The components of the solution include: @@ -1413,7 +1413,7 @@ def find_smallest_number(numbers): return smallest ``` -This function will correctly identify and return the smallest number from the given list.",6523,0.0 +This function will correctly identify and return the smallest number from the given list.",6523,0.0 2025-09-22T22:33:25.563613,[3],Please explain the different components of the solution for the following problem: Write a function to find the maximum difference between available pairs in the given tuple list.,8791.630299994722,,0e2ad5ff-b21e-421f-b5f9-4a1f4b413b58,Components of Solution," To solve this problem, we need to break down the task into several steps: @@ -1454,7 +1454,7 @@ def calculate_differences(pairs): #### Maximum Difference Identification ```python -def find",8763,0.0 +def find",8763,0.0 2025-09-22T22:33:31.016639,[3],Please explain the different components of the solution for the following problem: Write a function to sort a list of tuples using lambda.,5453.054500001599,,05a38f41-8f0e-4ed6-8602-715ac1deff6c,Sorting Tuples with Lambda," The solution involves creating a sorting function that utilizes the `sorted()` function along with a lambda function. The lambda function will specify how to compare the elements during the sorting process. @@ -1470,7 +1470,7 @@ tuples_list = [(1, 'a'), (2, 'b'), (3, 'c')] print(sort_tuples(tuples_list)) ``` -In this example, the lambda function `lambda x: x[0]` tells the `sorted()` function to sort the tuples based on their first element.",5427,0.0 +In this example, the lambda function `lambda x: x[0]` tells the `sorted()` function to sort the tuples based on their first element.",5427,0.0 2025-09-22T22:33:39.801561,[3],Please explain the different components of the solution for the following problem: Write a function of recursion list sum.,8784.25030001381,,57ddf55d-e79b-4408-8304-e06e0a040405,Recursion List Sum Components," The function `recursive_list_sum` takes an input list and returns the sum of all elements in that list using recursion. @@ -1498,7 +1498,7 @@ def recursive_list_sum(lst): return lst[0] + recursive_list_sum(lst[1:]) ``` -This function will correctly",8757,0.0 +This function will correctly",8757,0.0 2025-09-22T22:33:47.315199,[3],Please explain the different components of the solution for the following problem: Write a python function to count positive numbers in a list.,7513.608799999929,,73b07030-c2db-4917-ba2d-15d2143cef9d,Components of Solution," The solution involves creating a function that takes a list as input and returns the count of positive numbers within that list. Here are the key components: @@ -1521,7 +1521,7 @@ def count_positive_numbers(numbers): return count ``` -This function will correctly count the number of positive integers in the provided list.",7486,0.0 +This function will correctly count the number of positive integers in the provided list.",7486,0.0 2025-09-22T22:33:56.107243,[3],Please explain the different components of the solution for the following problem: Write a function to find the number of ways to partition a set of bell numbers.,8791.67010000674,,dddb693e-a0be-4e6b-ab4f-7c42932e7413,Bell Numbers Partitioning," The Bell numbers represent the number of ways to partition a set into non-empty subsets. To solve this problem, we need to understand how to generate Bell numbers and then count the partitions. @@ -1552,7 +1552,7 @@ To count the number of ways to partition a set using a given Bell number, we can import math def partition_count(k): - # This function calculates the number of ways to partition k elements into",8764,0.0 + # This function calculates the number of ways to partition k elements into",8764,0.0 2025-09-22T22:34:04.898829,[3],Please explain the different components of the solution for the following problem: Write a python function to check whether the given array is monotonic or not.,8790.728499996476,,a1d0a241-69b8-45f3-a2b7-8fb22fababae,Monotonic Array Check," The solution involves creating a function that takes an array as input and checks if it is either non-decreasing (increasing) or non-increasing (decreasing). Here are the steps: @@ -1572,7 +1572,7 @@ The solution involves creating a function that takes an array as input and check 5. **Return Result**: - If both `ascending` and `descending` remain False after the loop, return True indicating the array is neither increasing nor decreasing. - - Otherwise, return True if at least one of them is True; otherwise,",8764,0.0 + - Otherwise, return True if at least one of them is True; otherwise,",8764,0.0 2025-09-22T22:34:13.691392,[3],Please explain the different components of the solution for the following problem: Write a function to check whether a list contains the given sublist or not.,8792.26760000165,,100d9164-1852-4113-a65f-4a439b7a1753,List Sublist Checker," The solution involves creating a function that iterates through the main list and checks if any sublist matches the given sublist. Here are the key components: @@ -1597,7 +1597,7 @@ def check_sublist(main_list, sublist): return False # Test cases -print(check_sublist([1, 2, 3, 4], [2, 3])) # Output: True",8761,0.0 +print(check_sublist([1, 2, 3, 4], [2, 3])) # Output: True",8761,0.0 2025-09-22T22:34:22.480347,[3],Please explain the different components of the solution for the following problem: Write a function to find whether all the given tuples have equal length or not.,8789.264499995625,,9fabdd61-abf4-4c14-a9bc-60fb86566d3c,Tuple Length Checker," The different components of the solution include: @@ -1623,7 +1623,7 @@ def check_tuple_lengths(tuples_list): # Example usage print(check_tuple_lengths([(1, 2), (3, 4), (5, 6)])) # Output: True -print(check_tuple_lengths([(1, 2), (3, 4), (5)])) # Output: False",8763,0.0 +print(check_tuple_lengths([(1, 2), (3, 4), (5)])) # Output: False",8763,0.0 2025-09-22T22:34:31.274119,[3],Please explain the different components of the solution for the following problem: Write a function to sort a list of elements using comb sort.,8792.631999996956,,6ccb7110-416b-4f60-b842-3282f83af1bd,Comb Sort Components," Comb Sort is an efficient sorting algorithm that combines the advantages of insertion sort and binary search. Here are its main components: @@ -1652,7 +1652,7 @@ def comb_sort(arr): if arr[i] > arr[j]: arr[i], arr[j] = arr[j], arr[i] - # Insertion sort on the combined elements",8763,0.0 + # Insertion sort on the combined elements",8763,0.0 2025-09-22T22:34:40.066031,[3],Please explain the different components of the solution for the following problem: Write a python function to check whether the given number can be represented as difference of two squares or not.,8792.397800003528,,17c3ed2c-ef28-4db9-97ff-e9952471b37e,Components of Solution," To solve this problem, we need to understand that a number n can be represented as the difference of two squares if there exist integers x and y such that n = x^2 - y^2. @@ -1677,7 +1677,7 @@ def can_represent_as_difference_of_two_squares(n): return False for x in range(int(math.sqrt(n)) + 1): y = int(math.sqrt(x**2 + n)) - if y * y ==",8766,0.0 + if y * y ==",8766,0.0 2025-09-22T22:34:48.855695,[3],Please explain the different components of the solution for the following problem: Write a function to split the given string with multiple delimiters by using regex.,8789.074000000255,,2e7dfd4b-de29-42d8-9a12-df658278b32e,Regex String Splitter," The solution involves creating a function that uses regular expressions (regex) to split a string based on multiple delimiters. Here’s how you can do it: @@ -1704,7 +1704,7 @@ def split_string_with_regex(input_str, delimiters): # Split the input string based on the pattern result = re.split(pattern, input_str) - return result",8762,0.0 + return result",8762,0.0 2025-09-22T22:34:56.340035,[3],Please explain the different components of the solution for the following problem: Write a function to check whether it follows the sequence given in the patterns array.,7484.385999996448,,f4018c3a-cc91-4f30-bbeb-6116d463d2a1,Explain components of solution," The solution involves creating a function that checks if an input string matches any pattern defined in a list of patterns. Here are the key components: @@ -1728,7 +1728,7 @@ def check_pattern(input_string, patterns): return False ``` -This function will correctly determine if the input string follows any of the sequences provided in the patterns list.",7418,0.0 +This function will correctly determine if the input string follows any of the sequences provided in the patterns list.",7418,0.0 2025-09-22T22:35:05.135671,[3],Please explain the different components of the solution for the following problem: Write a function to find tuples which have all elements divisible by k from the given list of tuples.,8794.921799999429,,47bcbb71-831c-4c91-9a8d-df6012a1fe34,Tuple Divisibility Check," The solution involves creating a function that iterates through each tuple in the input list, checks if all elements within the tuple are divisible by a given number `k`, and returns a new list containing only those tuples that meet this condition. @@ -1752,7 +1752,7 @@ def find_divisible_tuples(tuples_list, k): # Initialize an empty list to store the results result = [] - # Iterate over each tuple in the",8766,0.0 + # Iterate over each tuple in the",8766,0.0 2025-09-22T22:35:13.938868,[3],Please explain the different components of the solution for the following problem: Write a python function to count the number of squares in a rectangle.,8803.012000003946,,b0cece80-4a17-4460-abcb-349a01de637c,Components of Solution," To solve this problem, we need to break down the task into several components: @@ -1783,7 +1783,7 @@ def count_squares_in_rectangle(width, height): print(count_squares_in_rectangle(5, 5)) # Output: 9 ``` -This function takes two parameters, `width` and `height`, checks their validity, calculates the area, determines the maximum possible side length of a square that fits inside the rectangle,",8761,0.0 +This function takes two parameters, `width` and `height`, checks their validity, calculates the area, determines the maximum possible side length of a square that fits inside the rectangle,",8761,0.0 2025-09-22T22:35:22.730626,[3],Please explain the different components of the solution for the following problem: Write a python function to find the difference between sum of even and odd digits.,8791.205500005162,,7fca5c19-4c7c-4760-99cd-747fd7001f48,Difference between sum of even and odd digits," The solution involves creating a function that takes an integer input, extracts its digits, categorizes them into even or odd, sums up each category, and returns the difference. @@ -1810,7 +1810,7 @@ def difference_of_sums(n): # Step 3: Iterate through each character (digit) in the string representation of the number for char in str_n: - digit = int",8763,0.0 + digit = int",8763,0.0 2025-09-22T22:35:31.521597,[3],Please explain the different components of the solution for the following problem: Write a python function to find number of integers with odd number of set bits.,8791.017299998202,,b0ac3017-8551-4b05-ad1d-f670bed61944,Components of Solution," The solution involves creating a function that counts the number of integers within a given range that have an odd number of set bits (1s) in their binary representation. @@ -1840,7 +1840,7 @@ Let's break down these steps into a more detailed plan: Use Python's built-in functions to convert the number to its binary form. 4. **Count Set Bits**: - Use the `bin()`",8762,0.0 + Use the `bin()`",8762,0.0 2025-09-22T22:35:40.313358,[3],Please explain the different components of the solution for the following problem: Write a python function to check whether the length of the word is odd or not.,8791.692500002682,,a56ff393-9e3d-4c3e-96f1-098c5cb189e7,Components of Solution," To solve this problem, we need to create a Python function that checks if the length of a given word is odd or even. Here are the main components of the solution: @@ -1868,7 +1868,7 @@ print(check_word_length(""hello"")) # Output: The length of 'hello' is odd. print(check_word_length(""world"")) # Output: The length of 'world' is even. ``` -This function can",8763,0.0 +This function can",8763,0.0 2025-09-22T22:35:47.208888,[3],Please explain the different components of the solution for the following problem: Write a function to find the nth tetrahedral number.,6895.167800001218,,fd482587-7e34-4931-b8fd-b3197bb20419,Tetrahedral Numbers," The tetrahedral numbers are defined by the formula T(n) = (n*(n+1)*(n+2))/6. To write a function to find the nth tetrahedral number, we need to follow these steps: @@ -1889,7 +1889,7 @@ def tetrahedral_number(n): print(tetrahedral_number(5)) ``` -This function will correctly compute the nth tetrahedral number using the given formula.",6867,0.0 +This function will correctly compute the nth tetrahedral number using the given formula.",6867,0.0 2025-09-22T22:35:54.068226,[3],Please explain the different components of the solution for the following problem: Write a function to zip the two given tuples.,6858.65229999763,,e2745a99-3a83-470c-8ceb-d52743df10f4,Zip Tuples Function," The solution involves creating a function that takes two tuples as input and returns a list of tuples containing elements from both input tuples. @@ -1908,7 +1908,7 @@ def zip_tuples(tuple1, tuple2): return zipped ``` -This function first checks if the lengths of the input tuples are equal. If not, it raises a `ValueError`. Then, it uses a list comprehension along with the built-in `zip` function to create a new list of tuples containing corresponding elements from the input tuples.",6831,0.0 +This function first checks if the lengths of the input tuples are equal. If not, it raises a `ValueError`. Then, it uses a list comprehension along with the built-in `zip` function to create a new list of tuples containing corresponding elements from the input tuples.",6831,0.0 2025-09-22T22:36:02.857985,[3],Please explain the different components of the solution for the following problem: Write a function to find the volume of a sphere.,8789.44130000309,,9131c8ed-da5e-4e2d-98ed-7680138e16a2,Sphere Volume Calculation," The formula to calculate the volume of a sphere is V = (4/3) * π * r^3, where r is the radius of the sphere. @@ -1937,7 +1937,7 @@ You can call this function with a specific radius value to get the volume of the print(sphere_volume(5)) # Output will be approximately 523.60 ``` -This function will correctly calculate the volume of any sphere given its radius.",8759,0.0 +This function will correctly calculate the volume of any sphere given its radius.",8759,0.0 2025-09-22T22:36:11.233677,[3],Please explain the different components of the solution for the following problem: Write a python function to find the character made by adding all the characters of the given string.,8375.472399988212,,03f54f2a-c395-4059-99ef-297c048516ee,Components of Solution," The solution involves creating a function that takes a string as input and returns the character formed by summing up the ASCII values of its characters. Here are the key steps: @@ -1963,7 +1963,7 @@ def find_character(input_string): print(find_character(""hello"")) # Outputs 'h' ``` -This function will correctly compute the desired character based on the sum of ASCII values of the input string.",8350,0.0 +This function will correctly compute the desired character based on the sum of ASCII values of the input string.",8350,0.0 2025-09-22T22:36:20.023463,[3],Please explain the different components of the solution for the following problem: Write a function to find the n-th number in newman conway sequence.,8789.89489999367,,c022f02b-deca-406b-95e5-789b62a102e2,Newman Conway Sequence," The Newman-Conway sequence is a sequence of numbers defined by the recurrence relation: @@ -1995,7 +1995,7 @@ def newman_conway(n): memo[x] = helper(helper(x - 1)) + 1 return memo[x] - return helper(n)",8762,0.0 + return helper(n)",8762,0.0 2025-09-22T22:36:28.707122,[3],Please explain the different components of the solution for the following problem: Write a function to find the surface area of a sphere.,8682.82950000139,,30addc72-7938-49df-8252-0c3834a454e8,Sphere Surface Area Calculation," The formula for the surface area (SA) of a sphere with radius r is given by SA = 4 * π * r^2. @@ -2030,7 +2030,7 @@ if __name__ == ""__main__"": print(f""The surface area of a sphere with radius {radius} is {calculate_surface_area(radius)}"") ``` -This function can be used to find the surface area of any sphere by providing the appropriate radius value.",8657,0.0 +This function can be used to find the surface area of any sphere by providing the appropriate radius value.",8657,0.0 2025-09-22T22:36:37.494237,[3],Please explain the different components of the solution for the following problem: Write a function to find nth centered hexagonal number.,8787.44880000886,,3a37ba8b-a4f0-435d-97ca-38ad374a95d1,Hexagonal Number Calculation," The hexagonal numbers are defined by the formula H(n) = 2n^2 - n. To calculate the nth centered hexagonal number, we need to adjust this formula slightly. @@ -2056,7 +2056,7 @@ def hexagonal_number(n): return (n * (n + 1)) // 2 # Test cases -print(hexagonal_number(1)) # Output should be",8759,0.0 +print(hexagonal_number(1)) # Output should be",8759,0.0 2025-09-22T22:36:46.279954,[3],Please explain the different components of the solution for the following problem: Write a function to merge three dictionaries into a single expression.,8785.087099997327,,e7d49343-f86b-4fa0-ad99-c3aabefdabed,Merging Dictionaries," To solve this problem, we can use several approaches depending on the version of Python you're using. @@ -2082,7 +2082,7 @@ print(merged_dict) # Output: {'a': 1, 'b': 2, 'c': 3} ### Using `|` Operator (Python 3.9+) ```python -def merge_dicts(dict",8758,0.0 +def merge_dicts(dict",8758,0.0 2025-09-22T22:36:54.137895,[3],Please explain the different components of the solution for the following problem: Write a function to get the frequency of the elements in a list.,7857.662499998696,,b7ab17d5-b792-47df-9228-523792b8df9b,Frequency Calculation in List," The different components of the solution for calculating the frequency of elements in a list can be broken down into the following steps: @@ -2108,7 +2108,7 @@ def get_frequency(lst): return frequency ``` -This function takes a list `lst` as input and returns a dictionary where keys are the elements from the list and values are their respective frequencies.",7830,0.0 +This function takes a list `lst` as input and returns a dictionary where keys are the elements from the list and values are their respective frequencies.",7830,0.0 2025-09-22T22:37:02.926570,[3],Please explain the different components of the solution for the following problem: Write a function to find the closest smaller number than n.,8788.53320000053,,e0262ccf-0984-4e3a-aea1-65d78e5c47dc,Closest Smaller Number Function," The solution involves creating a function that takes an integer `n` as input and returns the largest number less than or equal to `n` which has all its digits smaller than any digit in `n`. Here’s how we can approach this: @@ -2134,7 +2134,7 @@ def closest_smaller_number(n): for i in range(n-1, -1, -1): # Convert current number to string str_i = str(i) - if len(str_i) == length:",8758,0.0 + if len(str_i) == length:",8758,0.0 2025-09-22T22:37:11.333472,[3],Please explain the different components of the solution for the following problem: Write a python function to find the length of the longest word.,8406.998999998905,,6027b4e7-f83f-4c07-afd1-6917d8d52322,Components of Longest Word Length Function," The different components of the solution for finding the length of the longest word include: @@ -2166,7 +2166,7 @@ input_string = ""This is a sample sentence"" print(find_longest_word_length(input_string)) ``` -In this example, `find_longest_word_length` takes an input string, splits it into words, calculates their lengths, compares them to find the longest one, and returns its length.",8381,0.0 +In this example, `find_longest_word_length` takes an input string, splits it into words, calculates their lengths, compares them to find the longest one, and returns its length.",8381,0.0 2025-09-22T22:37:20.130061,[3],Please explain the different components of the solution for the following problem: Write a function to check if a substring is present in a given list of string values.,8795.946500002174,,aac456bc-11b2-4858-ad9b-6384ba4cdc8f,Components of Solution," The solution involves creating a function that iterates through each element in the provided list and checks if the substring exists within each element. The function should return True if the substring is found in any of the elements, otherwise False. @@ -2189,7 +2189,7 @@ def check_substring(substring, list_of_strings): return False ``` -This function will efficiently check for the presence of a substring in a list of strings.",8764,0.0 +This function will efficiently check for the presence of a substring in a list of strings.",8764,0.0 2025-09-22T22:37:28.918806,[3],Please explain the different components of the solution for the following problem: Write a function to check whether the given number is undulating or not.,8788.044300003094,,dd415da9-e6d7-4cdf-a9e9-35091dc952c5,Components of Solution," To determine if a number is undulating, we need to evaluate its digits' behavior relative to their neighbors. Here’s how you can break down the solution into components: @@ -2222,7 +2222,7 @@ Compare each digit with its neighbors to determine if the sequence forms an undu ```python def is_undulating(digits): for i in range(1, len(digits) - 1): - if (digits[i] > digits[i -",8762,0.0 + if (digits[i] > digits[i -",8762,0.0 2025-09-22T22:37:37.710247,[3],Please explain the different components of the solution for the following problem: Write a function to calculate the value of 'a' to the power 'b'.,8790.960500002257,,7713f204-a5d7-4aa5-a565-564d957c8f20,Power Calculation Function," The different components of the solution include: @@ -2246,7 +2246,7 @@ def power(a, b): # Test cases print(power(2, 3)) # Output: 8 -print(power(2, -3)) # Output: 0.",8763,0.0 +print(power(2, -3)) # Output: 0.",8763,0.0 2025-09-22T22:37:46.504017,[3],Please explain the different components of the solution for the following problem: Write a function to extract the index minimum value record from the given tuples.,8794.086699999752,,120da51b-994c-43c1-94e2-9ceff296d349,Extracting Minimum Value Record," To solve this problem, we need to create a function that takes a list of tuples as input and returns the tuple with the minimum value. Here's how we can break down the solution into its components: @@ -2271,7 +2271,7 @@ def extract_minimum_value_record(tuples_list): # Step 3 & 4: Iteration and Comparison for t in tuples_list: - if",8762,0.0 + if",8762,0.0 2025-09-22T22:37:55.291954,[3],Please explain the different components of the solution for the following problem: Write a python function to find the minimum length of sublist.,8787.444000001415,,c6e26558-0c28-4d0f-a2a7-a621e9a31195,Components of Solution for Minimum Length Sublist," The solution involves several steps: @@ -2305,7 +2305,7 @@ def min_length_sublist(lst): # Step 4: Validation if all(elem in current_sub_list for elem in lst): - #",8760,0.0 + #",8760,0.0 2025-09-22T22:38:04.081618,[3],Please explain the different components of the solution for the following problem: Write a python function to find the number of divisors of a given integer.,8789.503699998022,,cac23e5f-1ede-48f4-953a-bc6035bea482,Divisor Function Components," To solve this problem, we need to create a Python function that calculates the number of divisors of a given integer. Here are the key components of the solution: @@ -2329,7 +2329,7 @@ def count_divisors(n): count = 0 # Loop through possible divisors - for i in range(1, int(n**0.5) + 1):",8762,0.0 + for i in range(1, int(n**0.5) + 1):",8762,0.0 2025-09-22T22:38:12.869129,[3],Please explain the different components of the solution for the following problem: Write a function to find frequency count of list of lists.,8787.306199999875,,1eb756b4-5fe3-4f42-b6b4-253e1f06e6cc,Frequency Count of List of Lists," To solve this problem, we need to create a function that can handle nested lists and calculate the frequency count of each element within those lists. @@ -2361,7 +2361,7 @@ def frequency_count_of_lists(lst): flat_list = flatten_list(lst) # Step 2: Count frequencies using Counter - freq_count = Counter(",8761,0.0 + freq_count = Counter(",8761,0.0 2025-09-22T22:38:19.938135,[3],Please explain the different components of the solution for the following problem: Write a function to multiply all the numbers in a list and divide with the length of the list.,7068.734699991182,,03d63806-b82b-4f42-a8fe-d56a956c9155,Multiply and Divide List Numbers," To solve this problem, we need to perform two main operations on the list of numbers: @@ -2389,7 +2389,7 @@ numbers = [1, 2, 3, 4] print(multiply_and_divide(numbers)) # Output: 3.0 ``` -This function handles an empty list by returning `None` and performs the required multiplication and division operations.",7042,0.0 +This function handles an empty list by returning `None` and performs the required multiplication and division operations.",7042,0.0 2025-09-22T22:38:28.071869,[3],Please explain the different components of the solution for the following problem: Write a function to convert the given decimal number to its binary equivalent.,8133.280199996079,,2fb06d83-3c05-4d9f-bc8a-9dfcd0812654,Decimal to Binary Conversion," The solution involves creating a function that takes a decimal number as input and returns its binary equivalent. Here are the main steps: @@ -2418,7 +2418,7 @@ def decimal_to_binary(decimal_number): return binary_representation ``` -You can test this function with various decimal numbers to see how it works.",8105,0.0 +You can test this function with various decimal numbers to see how it works.",8105,0.0 2025-09-22T22:38:36.653422,[3],Please explain the different components of the solution for the following problem: Write a function to find the next smallest palindrome of a specified number.,8581.620400000247,,46a8f8d3-08ce-4b23-b621-828750ce330a,Components of Palindrome Function," The solution involves several key components: @@ -2450,7 +2450,7 @@ def next_smallest_palindrome(n): print(next_smallest_palindrome(10)) # Output: 11 ``` -This approach ensures that the function efficiently finds the next smallest palindrome by checking each subsequent number until a palindrome is encountered.",8555,0.0 +This approach ensures that the function efficiently finds the next smallest palindrome by checking each subsequent number until a palindrome is encountered.",8555,0.0 2025-09-22T22:38:45.267721,[3],Please explain the different components of the solution for the following problem: Write a function to find the kth element in the given array.,8614.021900008083,,cc2fb896-c17d-449e-845c-689bcc65585d,Explanation of Solution Components," The solution can be implemented using various approaches such as sorting, binary search, or using a heap data structure. Here are the main components of the solution: @@ -2473,7 +2473,7 @@ The solution can be implemented using various approaches such as sorting, binary - Handle cases when `k` is larger than the length of the array. - Handle cases when the array contains duplicate values. -Each approach has its own advantages and trade-offs in terms of time complexity and space complexity. The choice of approach depends on the specific requirements and constraints of the problem.",8589,0.0 +Each approach has its own advantages and trade-offs in terms of time complexity and space complexity. The choice of approach depends on the specific requirements and constraints of the problem.",8589,0.0 2025-09-22T22:38:52.233116,[3],Please explain the different components of the solution for the following problem: Write a function to convert snake case string to camel case string.,6964.210300007835,,36699c1a-812b-493f-8db3-cb387a4dc866,Snake Case to Camel Case Conversion," The function should take a string as input and return a string with the first letter capitalized and the rest of the words separated by spaces. @@ -2497,7 +2497,7 @@ def snake_case_to_camel_case(snake_str): This function works by splitting the input string at underscores, capitalizing the first letter of each subsequent word (except the first), and then joining them back together without any separators. Let me know if you have any questions or need further clarification! -```",6936,0.0 +```",6936,0.0 2025-09-22T22:39:01.019179,[3],"Please explain the different components of the solution for the following problem: Write a function to find eulerian number a(n, m).",8786.31700000551,,2f998997-19e6-4ba6-a2cb-cd515fa99243,Eulerian Number Components," The Eulerian number a(n, m) can be calculated using the following steps: @@ -2515,7 +2515,7 @@ def eulerian_number(a, b): if a == 0 or b == 0: return 1 elif a > 0 and b > 0: - return sum(eulerian_number(a-1, b), eulerian_number(a-1, b+",8761,0.0 + return sum(eulerian_number(a-1, b), eulerian_number(a-1, b+",8761,0.0 2025-09-22T22:39:09.811272,[3],Please explain the different components of the solution for the following problem: Write a function to sort each sublist of strings in a given list of lists using lambda function.,8791.95109999273,,b2f50018-1874-4175-8ada-f20a634b7025,Sorting Sublists with Lambda Function," To solve this problem, we need to create a function that will take a list of lists (where each sublist contains strings) and sort each sublist individually using a lambda function. @@ -2537,7 +2537,7 @@ def sort_sublists(lists): return [sorted(sublist, key=lambda x: x) for sublist in lists] # Example usage -sublists = [['banana', 'apple', '",8766,0.0 +sublists = [['banana', 'apple', '",8766,0.0 2025-09-22T22:39:16.634778,[3],Please explain the different components of the solution for the following problem: Write a python function to count true booleans in the given list.,6822.8661000030115,,e44a5bd7-58cc-4788-96aa-b45617b7a153,Count True Booleans in List," The solution involves creating a function that iterates through the list and counts the number of `True` values. Here are the steps: @@ -2566,7 +2566,7 @@ def count_true_booleans(lst): return sum(1 for item in lst if item) ``` -Both implementations will give you the desired result.",6798,0.0 +Both implementations will give you the desired result.",6798,0.0 2025-09-22T22:39:24.661233,[3],Please explain the different components of the solution for the following problem: Write a function to add the given list to the given tuples.,8025.697299992316,,7d8f39d8-0bb3-4f1e-9bff-a43cc55d7a3d,Components of Solution," The solution involves creating a function that takes two parameters: a list and a tuple. The function will iterate over each element in the list and append it to the corresponding position in the tuple. Here are the steps: @@ -2591,7 +2591,7 @@ def add_list_to_tuple(lst, tpl): return tpl ``` -This function assumes that both the list and the tuple have the same length. If they do not, it raises a `ValueError`.",8000,0.0 +This function assumes that both the list and the tuple have the same length. If they do not, it raises a `ValueError`.",8000,0.0 2025-09-22T22:39:33.454927,[3],Please explain the different components of the solution for the following problem: Write a python function to count hexadecimal numbers for a given range.,8794.12510000111,,1686cd4a-6d32-4641-a33a-6acd30888cbc,Components of Hex Count Function," The different components of the solution for counting hexadecimal numbers within a given range include: @@ -2621,7 +2621,7 @@ def count_hex_digits(start, end): print(count_hex_digits(1, 10)) # Output should be the count of hexadecimal digits from 1 to 10 ``` -This function will correctly count all hex",8761,0.0 +This function will correctly count all hex",8761,0.0 2025-09-22T22:39:42.243408,[3],Please explain the different components of the solution for the following problem: Write a function to merge multiple sorted inputs into a single sorted iterator using heap queue algorithm.,8788.167599996086,,9c015f7e-2b55-4c2c-8c4d-fe677b79f297,Merging Sorted Inputs with Heap Queue Algorithm," To solve this problem, we need to create a function that merges multiple sorted inputs into a single sorted iterator. The heap queue (priority queue) algorithm can be used here because it efficiently handles merging operations. @@ -2643,7 +2643,7 @@ Here’s a sample implementation in Python: ```python import heapq -def merge",8763,0.0 +def merge",8763,0.0 2025-09-22T22:39:51.033882,[3],Please explain the different components of the solution for the following problem: Write a python function to find the count of rotations of a binary string with odd value.,8789.735199999996,,62044b0b-8f6e-4af1-a2d3-5e42a5aa2637,Components of Solution," To solve this problem, we need to break down the task into several steps: @@ -2669,7 +2669,7 @@ def count_rotations(binary_str): # Step 2: Count rotations count = 0 for i in range(n): - # Step 3: Efficiently check for rotations using bitwise operations",8763,0.0 + # Step 3: Efficiently check for rotations using bitwise operations",8763,0.0 2025-09-22T22:39:59.828327,[3],Please explain the different components of the solution for the following problem: Write a function to extract the ranges that are missing from the given list with the given start range and end range values.,8794.587100011995,,5a65fb17-a320-4b72-992d-84d36791c202,Extracting Missing Ranges," To solve this problem, we need to identify the gaps between consecutive elements in the given list. Here are the steps to achieve this: @@ -2698,4 +2698,4 @@ def extract_missing_ranges(lst, start_range, end_range): prev_end = None missing_ranges = [] - for i in range(len(lst",8769,0.0 + for i in range(len(lst",8769,0.0 diff --git a/src/privacy/__init__.py b/src/privacy/__init__.py new file mode 100644 index 0000000..1d69337 --- /dev/null +++ b/src/privacy/__init__.py @@ -0,0 +1,8 @@ +"""Account-level privacy controls (GDPR): data-collection consent and erasure. + +``collection`` decides whether data about an account may be stored at all and +records an opt-out; ``erasure`` removes what was collected. Every collection +path asks ``collection`` rather than trusting a client flag. Like the research +stores, both modules take a caller-managed SQLAlchemy session and never commit, +so the router owns the unit of work. +""" diff --git a/src/privacy/collection.py b/src/privacy/collection.py new file mode 100644 index 0000000..28d09b5 --- /dev/null +++ b/src/privacy/collection.py @@ -0,0 +1,162 @@ +"""Whether an account's data may be collected, and the opt-out that stops it. + +The opt-out lives in ``user.data_collection_opted_out_at`` rather than in the +preference JSON: clients rewrite that JSON wholesale from their local state, so +a stale IDE would otherwise undo a withdrawal nobody asked it to undo. Only the +privacy endpoints write the column. + +While an account is opted out nothing is collected about it, whatever a client +sends: the classic handlers persist nothing, agent content and agent memory are +refused, and it holds no ACTIVE study enrollment (the opt-out withdraws it and +joining refuses until the account opts back in). + +Writers that race an opt-out or erase check with :func:`lock_collection_allowed`: +its share lock conflicts with the opt-out's update of the account row, so a +write either commits before the erase starts (and is erased with the rest) or +waits for it and then sees the opt-out. +""" + +from __future__ import annotations + +import json +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Optional + +from sqlalchemy import delete, func, select, update + +from database.db_schemas import BehavioralTelemetry, Context, ContextualTelemetry, User +from research.participants import identity as participant_identity + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + +def is_collection_allowed(db: Session, user_id: uuid.UUID) -> bool: + """Whether data about ``user_id`` may be stored now (False for unknown accounts).""" + row = db.execute( + select(User.data_collection_opted_out_at).where(User.user_id == user_id) + ).first() + return row is not None and row[0] is None + + +def opted_out_at(db: Session, user_id: uuid.UUID) -> Optional[datetime]: + """When the account opted out of data collection, or ``None`` while it collects.""" + return db.execute( + select(User.data_collection_opted_out_at).where(User.user_id == user_id) + ).scalar_one_or_none() + + +def lock_collection_allowed(db: Session, user_id: uuid.UUID) -> bool: + """:func:`is_collection_allowed`, holding a share lock on the account row. + + The lock lasts until the caller's transaction ends. An opt-out (and so an + erase) updates the same row first, so the two serialize: either the opt-out + commits first and this returns False, or the opt-out waits until the + caller's write has committed and then withdraws or erases it. Study joins + and the storage writers that can race an erase take it. + """ + row = db.execute( + select(User.data_collection_opted_out_at) + .where(User.user_id == user_id) + .with_for_update(read=True) + ).first() + return row is not None and row[0] is None + + +def lock_account(db: Session, user_id: uuid.UUID) -> None: + """Share-lock the account row until the transaction ends, whatever its opt-out state. + + For writers that stay allowed while an account is opted out (content-free + agent bookkeeping) but must not slip past a concurrent erase; see + :func:`lock_collection_allowed`. + """ + db.execute(select(User.user_id).where(User.user_id == user_id).with_for_update(read=True)) + + +def _context_storage_allowed(opted_out_at: Optional[datetime], preference: Optional[str]) -> bool: + if opted_out_at is not None: + return False + try: + parsed = json.loads(preference or "{}") + except (TypeError, ValueError): + return False + return isinstance(parsed, dict) and bool(parsed.get("store_context", False)) + + +def allows_context_storage(user: User) -> bool: + """Whether ``user`` lets their project's code context be persisted.""" + return _context_storage_allowed(user.data_collection_opted_out_at, user.preference) + + +def lock_context_storage_allowed(db: Session, user_id: uuid.UUID) -> bool: + """:func:`allows_context_storage` for ``user_id``, under the account-row share lock.""" + row = db.execute( + select(User.data_collection_opted_out_at, User.preference) + .where(User.user_id == user_id) + .with_for_update(read=True) + ).first() + return row is not None and _context_storage_allowed(row[0], row[1]) + + +def discard_if_opted_out( + db: Session, + user_id: uuid.UUID, + *, + context_id: Optional[uuid.UUID] = None, + contextual_telemetry_id: Optional[uuid.UUID] = None, + behavioral_telemetry_id: Optional[uuid.UUID] = None, +) -> bool: + """Write-time check for a request accepted before the account opted out. + + Handlers check before they enqueue their storage tasks; the tasks check + again, under :func:`lock_collection_allowed`, right before they write. When + collection is allowed, False is returned and the lock is held until the + caller's next commit, which carries its top-level row (a query): rows written + after that commit hang off it and go with it by cascade, or fail their + foreign key if it was erased meanwhile. Otherwise the request's context and + telemetry rows (stored by an earlier task) are deleted and True is returned: + the caller stores nothing more. + """ + if lock_collection_allowed(db, user_id): + return False + for model, column, row_id in ( + (Context, Context.context_id, context_id), + (ContextualTelemetry, ContextualTelemetry.contextual_telemetry_id, contextual_telemetry_id), + (BehavioralTelemetry, BehavioralTelemetry.behavioral_telemetry_id, behavioral_telemetry_id), + ): + if row_id is not None: + db.execute(delete(model).where(column == row_id)) + return True + + +def opt_out( + db: Session, user_id: uuid.UUID, *, now: Optional[datetime] = None +) -> list[uuid.UUID]: + """Withdraw consent: stop collection and end the active study enrollment. + + Idempotent: a repeated opt-out keeps the original timestamp. Returns the ids + of the enrollments it withdrew. + """ + timestamp = now or datetime.now(timezone.utc) + # Update the account row before touching enrollments: its row lock is what + # serializes the opt-out with a concurrent join (see lock_collection_allowed). + db.execute( + update(User) + .where(User.user_id == user_id) + .values( + data_collection_opted_out_at=func.coalesce( + User.data_collection_opted_out_at, timestamp + ) + ) + ) + return participant_identity.withdraw_active_enrollments(db, user_id, now=timestamp) + + +def opt_in(db: Session, user_id: uuid.UUID) -> None: + """Resume collection under the account's preferences; it does not re-enroll.""" + db.execute( + update(User) + .where(User.user_id == user_id) + .values(data_collection_opted_out_at=None) + ) diff --git a/src/privacy/erasure.py b/src/privacy/erasure.py new file mode 100644 index 0000000..be11afd --- /dev/null +++ b/src/privacy/erasure.py @@ -0,0 +1,246 @@ +"""Erase what Code4Me collected about an account (GDPR Art. 17). + +``erase_collected_data`` backs the self-service "erase my data" action. It opts +the account out and deletes every row of collected data linked to it, in one +transaction and in foreign-key order: + +* agent runs (tasks with their events and edits) and agent memory; +* research participation and telemetry (``research.participants.erasure``); +* classic completion/chat requests with their generations, ground truth, code + context and telemetry; chats; study config assignments; ended sessions. + +It keeps the account, the live sign-in session (the IDE cannot re-acquire a +revoked one) and the project rows, whose stored code context it clears. Studies +and agent profiles the account authored are research resources, not data about +the account, and stay. + +``delete_account`` then also removes the remaining sessions, project +memberships, sole-member projects and the account row. It refuses accounts that +own studies or agent profiles (``account_deletion_blocker``): deleting those +would cascade into other people's study data. +""" + +from __future__ import annotations + +import uuid +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Collection, Optional + +from sqlalchemy import delete, func, or_, select, update + +from database.db_schemas import ( + AgentMemory, + AgentProfile, + AgentTask, + BehavioralTelemetry, + Chat, + ConfigAssignmentHistory, + Context, + ContextualTelemetry, + MetaQuery, + Project, + ProjectUser, + Study, + User, +) +from database.db_schemas import Session as SessionRow +from research.participants import erasure as participant_erasure + +from . import collection + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + +# Bound on the ids bound into one IN (...) clause. +_ID_CHUNK = 1000 + + +@dataclass(frozen=True) +class StoredData: + """Collected data held for an account, by category (also the erasure report).""" + + queries: int = 0 + chats: int = 0 + agent_runs: int = 0 + study_enrollments: int = 0 + study_events: int = 0 + + def as_dict(self) -> dict[str, int]: + return asdict(self) + + +def _count(db: Session, model, *criteria) -> int: + return int(db.execute(select(func.count()).select_from(model).where(*criteria)).scalar_one()) + + +def _agent_run_scope(user_id: uuid.UUID): + # A run belongs to the account that ran it; older rows may carry only the + # account's classic session. + return or_( + AgentTask.owner_user_id == user_id, + AgentTask.session_id.in_(select(SessionRow.session_id).where(SessionRow.user_id == user_id)), + ) + + +def stored_data_summary(db: Session, user_id: uuid.UUID) -> StoredData: + """Count the collected data currently held for the account.""" + research = participant_erasure.participant_data_counts(db, user_id) + return StoredData( + queries=_count(db, MetaQuery, MetaQuery.user_id == user_id), + chats=_count(db, Chat, Chat.user_id == user_id), + agent_runs=_count(db, AgentTask, _agent_run_scope(user_id)), + study_enrollments=research.enrollments, + study_events=research.events, + ) + + +class AccountDeletionBlocked(Exception): + """The account owns research resources and cannot delete itself.""" + + +def account_deletion_blocker(db: Session, user_id: uuid.UUID) -> Optional[str]: + """Why the account cannot delete itself, or ``None`` when it can.""" + owned = [] + studies = _count(db, Study, Study.created_by == user_id) + if studies: + owned.append(f"{studies} research {'study' if studies == 1 else 'studies'}") + profiles = _count(db, AgentProfile, AgentProfile.owner_user_id == user_id) + if profiles: + owned.append(f"{profiles} agent {'profile' if profiles == 1 else 'profiles'}") + if not owned: + return None + return ( + f"This account owns {' and '.join(owned)}, which other people's study data can " + "depend on, so it cannot delete itself. Ask an administrator to remove them first." + ) + + +def _delete_ids(db: Session, model, column, ids: list) -> None: + ids = [value for value in ids if value is not None] + for start in range(0, len(ids), _ID_CHUNK): + db.execute(delete(model).where(column.in_(ids[start : start + _ID_CHUNK]))) + + +def _erase_agent_data(db: Session, user_id: uuid.UUID) -> int: + # A task cascades to its events and edits. + runs = db.execute(delete(AgentTask).where(_agent_run_scope(user_id))).rowcount + db.execute(delete(AgentMemory).where(AgentMemory.owner_user_id == str(user_id))) + return runs + + +def _erase_classic_data(db: Session, user_id: uuid.UUID) -> tuple[int, int]: + references = db.execute( + select( + MetaQuery.context_id, + MetaQuery.contextual_telemetry_id, + MetaQuery.behavioral_telemetry_id, + ).where(MetaQuery.user_id == user_id) + ).all() + # A query cascades to its completion/chat query, generations and ground + # truth. Its context and telemetry rows are referenced without a cascade, so + # they can only go once the query is gone. + queries = db.execute(delete(MetaQuery).where(MetaQuery.user_id == user_id)).rowcount + _delete_ids(db, Context, Context.context_id, [row.context_id for row in references]) + _delete_ids( + db, + ContextualTelemetry, + ContextualTelemetry.contextual_telemetry_id, + [row.contextual_telemetry_id for row in references], + ) + _delete_ids( + db, + BehavioralTelemetry, + BehavioralTelemetry.behavioral_telemetry_id, + [row.behavioral_telemetry_id for row in references], + ) + chats = db.execute(delete(Chat).where(Chat.user_id == user_id)).rowcount + db.execute(delete(ConfigAssignmentHistory).where(ConfigAssignmentHistory.user_id == user_id)) + return queries, chats + + +def _clear_project_context(db: Session, user_id: uuid.UUID) -> None: + """Clear the stored code context of the account's projects. + + A shared project keeps its context while another member still lets context + be stored: it is that member's code as much as this account's. + """ + project_ids = db.execute( + select(ProjectUser.project_id).where(ProjectUser.user_id == user_id) + ).scalars().all() + to_clear = [] + for project_id in project_ids: + other_members = db.execute( + select(User) + .join(ProjectUser, ProjectUser.user_id == User.user_id) + .where(ProjectUser.project_id == project_id, User.user_id != user_id) + ).scalars().all() + if not any(collection.allows_context_storage(member) for member in other_members): + to_clear.append(project_id) + if to_clear: + db.execute( + update(Project) + .where(Project.project_id.in_(to_clear)) + .values(multi_file_contexts="{}", multi_file_context_changes="{}") + ) + + +def erase_collected_data( + db: Session, + user_id: uuid.UUID, + *, + keep_session_ids: Collection[uuid.UUID] = (), + now: Optional[datetime] = None, +) -> StoredData: + """Opt the account out and erase everything collected about it. + + ``keep_session_ids`` names live sign-in sessions to keep. Returns what was + erased. + """ + timestamp = now or datetime.now(timezone.utc) + collection.opt_out(db, user_id, now=timestamp) + db.flush() + # Runs and queries reference sessions without a cascade: sessions go last. + agent_runs = _erase_agent_data(db, user_id) + research = participant_erasure.erase_participant_data(db, user_id, now=timestamp) + queries, chats = _erase_classic_data(db, user_id) + _clear_project_context(db, user_id) + sessions = delete(SessionRow).where(SessionRow.user_id == user_id) + if keep_session_ids: + sessions = sessions.where(SessionRow.session_id.not_in(list(keep_session_ids))) + db.execute(sessions) + return StoredData( + queries=queries, + chats=chats, + agent_runs=agent_runs, + study_enrollments=research.enrollments, + study_events=research.events, + ) + + +def delete_account( + db: Session, user_id: uuid.UUID, *, now: Optional[datetime] = None +) -> StoredData: + """Erase everything collected about the account, then the account itself. + + Raises :class:`AccountDeletionBlocked` for an account that owns studies or + agent profiles. Returns what was erased. + """ + # Lock the account row first: rows that reference it (a new study, say) + # cannot appear between the ownership check and the delete. + db.execute(select(User.user_id).where(User.user_id == user_id).with_for_update()) + blocker = account_deletion_blocker(db, user_id) + if blocker is not None: + raise AccountDeletionBlocked(blocker) + erased = erase_collected_data(db, user_id, now=now) + own_projects = select(ProjectUser.project_id).where(ProjectUser.user_id == user_id) + shared_projects = select(ProjectUser.project_id).where(ProjectUser.user_id != user_id) + db.execute( + delete(Project).where( + Project.project_id.in_(own_projects), + Project.project_id.not_in(shared_projects), + ) + ) + # Remaining memberships cascade with the account row. + db.execute(delete(User).where(User.user_id == user_id)) + return erased diff --git a/src/research/__init__.py b/src/research/__init__.py new file mode 100644 index 0000000..89682e3 --- /dev/null +++ b/src/research/__init__.py @@ -0,0 +1,8 @@ +"""Research platform core packages. + +This package is intentionally backend-agnostic: it must never import ``App``, +FastAPI, or a SQLAlchemy session factory. The API layer +(``backend.routers.research``) supplies sockets and persistence; the core +modules here only hold domain models, deterministic serialization, redaction, +receipt building/validation and the compatibility evaluator. +""" diff --git a/src/research/analysis/__init__.py b/src/research/analysis/__init__.py new file mode 100644 index 0000000..b27ac09 --- /dev/null +++ b/src/research/analysis/__init__.py @@ -0,0 +1,11 @@ +"""Researcher analysis: read models and pilot operations. + +Merged home for the scoped researcher read models/RBAC and pilot operations +(health, release gate, kill switch and retention). The individual sub-packages +keep their original public surfaces: + +* :mod:`research.analysis.read_models` +* :mod:`research.analysis.operations` + +The core packages never import ``App``, FastAPI, or a session factory. +""" diff --git a/src/research/analysis/operations/__init__.py b/src/research/analysis/operations/__init__.py new file mode 100644 index 0000000..f79fd0d --- /dev/null +++ b/src/research/analysis/operations/__init__.py @@ -0,0 +1,99 @@ +"""Pilot operations, retention drills, and the release gate (Issue 13). + +Public surface: + +* :mod:`research.analysis.operations.enums` - release/pilot/health/retention vocabularies + and typed reason codes. +* :mod:`research.analysis.operations.models` - ``OperationalHealthV1`` (metadata only), + immutable ``ReleaseEvidenceV1``, ``PilotRunV1``, kill-switch and retention + verification contracts. +* :mod:`research.analysis.operations.health` - pure health aggregation with explicit + ``UNKNOWN`` on a monitoring outage (health is never invented). +* :mod:`research.analysis.operations.release_gate` - ``evaluate_release``. +* :mod:`research.analysis.operations.kill_switch` - auditable scoped kill switch and the + injected predicate used by bootstrap/ingestion. +* :mod:`research.analysis.operations.retention` - deletion-drill verification. +* :mod:`research.analysis.operations.store` - Session-supplied persistence helpers. + +The core package never imports ``App``, FastAPI, or a session factory. +""" + +from . import store +from .enums import ( + EvidenceState, + HealthSignalState, + KillSwitchScopeKind, + OperationsReasonCode, + PilotClassification, + PilotOutcome, + ReleaseDecision, + RetentionVerificationStatus, +) +from .health import HealthInputs, HealthThresholds, aggregate_health, health_summary +from .kill_switch import ( + KillSwitchRegistry, + engage_kill_switch, + is_engaged, + kill_switch_check, + kill_switch_issue, + release_kill_switch, +) +from .models import ( + ComponentEvidence, + DeleteVerification, + FaultScenario, + HealthSignal, + IncidentRecord, + KillSwitchRecord, + KillSwitchScope, + OperationalHealthV1, + OperationsIssue, + PilotRunV1, + ReleaseEvaluationResult, + ReleaseEvidenceV1, + ReleaseReason, + RetentionVerification, + ReviewerApproval, +) +from .release_gate import evaluate_release, record_decision +from .retention import verify_deletion + +__all__ = [ + "ComponentEvidence", + "DeleteVerification", + "EvidenceState", + "FaultScenario", + "HealthInputs", + "HealthSignal", + "HealthSignalState", + "HealthThresholds", + "IncidentRecord", + "KillSwitchRecord", + "KillSwitchRegistry", + "KillSwitchScope", + "KillSwitchScopeKind", + "OperationalHealthV1", + "OperationsIssue", + "OperationsReasonCode", + "PilotClassification", + "PilotOutcome", + "PilotRunV1", + "ReleaseDecision", + "ReleaseEvaluationResult", + "ReleaseEvidenceV1", + "ReleaseReason", + "RetentionVerification", + "RetentionVerificationStatus", + "ReviewerApproval", + "aggregate_health", + "engage_kill_switch", + "evaluate_release", + "health_summary", + "is_engaged", + "kill_switch_check", + "kill_switch_issue", + "record_decision", + "release_kill_switch", + "store", + "verify_deletion", +] diff --git a/src/research/analysis/operations/enums.py b/src/research/analysis/operations/enums.py new file mode 100644 index 0000000..c0f30d0 --- /dev/null +++ b/src/research/analysis/operations/enums.py @@ -0,0 +1,117 @@ +"""Closed vocabularies for pilot operations, retention and the release gate. + +Every member is part of a persisted contract (health rows, release evidence, +kill-switch records, retention verifications); members are additive only. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = [ + "EvidenceState", + "HealthSignalState", + "KillSwitchScopeKind", + "OperationsReasonCode", + "PilotClassification", + "PilotOutcome", + "ReleaseDecision", + "RetentionVerificationStatus", +] + + +class ReleaseDecision(str, Enum): + """Terminal outcome of the release gate.""" + + GO = "GO" + NO_GO = "NO_GO" + GO_WITH_LIMITS = "GO_WITH_LIMITS" + EXPIRED = "EXPIRED" + + +class PilotClassification(str, Enum): + """Whether a pilot ran on synthetic or consented data.""" + + SYNTHETIC = "SYNTHETIC" + CONSENTED = "CONSENTED" + + +class HealthSignalState(str, Enum): + """State of one operational health signal.""" + + HEALTHY = "HEALTHY" + DEGRADED = "DEGRADED" + UNKNOWN = "UNKNOWN" + CRITICAL = "CRITICAL" + + +class PilotOutcome(str, Enum): + """Terminal outcome of a pilot run.""" + + COMPLETED = "COMPLETED" + COMPLETED_WITH_INCIDENTS = "COMPLETED_WITH_INCIDENTS" + FAILED = "FAILED" + ABORTED = "ABORTED" + + +class EvidenceState(str, Enum): + """Presence/validity of one required component or evidence item.""" + + PRESENT = "PRESENT" + MISSING = "MISSING" + UNKNOWN = "UNKNOWN" + EXPIRED = "EXPIRED" + CHANGED = "CHANGED" + + +class KillSwitchScopeKind(str, Enum): + """The scope a kill switch applies to.""" + + STUDY = "STUDY" + ENROLLMENT = "ENROLLMENT" + + +class RetentionVerificationStatus(str, Enum): + """Result of a deletion-retention drill.""" + + VERIFIED = "VERIFIED" + PARTIAL = "PARTIAL" + FAILED = "FAILED" + + +class OperationsReasonCode(str, Enum): + """Stable machine-readable reason for an operations/gate outcome.""" + + OK = "OK" + + # Health signals. + MONITORING_OUTAGE = "MONITORING_OUTAGE" + MISSING_HEALTH_INPUT = "MISSING_HEALTH_INPUT" + THRESHOLD_DEGRADED = "THRESHOLD_DEGRADED" + THRESHOLD_CRITICAL = "THRESHOLD_CRITICAL" + SPOOL_BACKLOG = "SPOOL_BACKLOG" + UPLOAD_ACK_LOW = "UPLOAD_ACK_LOW" + RUNTIME_EXIT_SPIKE = "RUNTIME_EXIT_SPIKE" + CAPABILITY_MISMATCH = "CAPABILITY_MISMATCH" + RETENTION_INCOMPLETE = "RETENTION_INCOMPLETE" + EXPORT_UNHEALTHY = "EXPORT_UNHEALTHY" + SESSION_ANOMALY = "SESSION_ANOMALY" + + # Release gate. + REQUIRED_COMPONENT_MISSING = "REQUIRED_COMPONENT_MISSING" + REQUIRED_EVIDENCE_MISSING = "REQUIRED_EVIDENCE_MISSING" + COMPONENT_RECEIPT_EXPIRED = "COMPONENT_RECEIPT_EXPIRED" + CONFIG_EXPIRED = "CONFIG_EXPIRED" + MATERIAL_CHANGE = "MATERIAL_CHANGE" + LIMITATIONS_PRESENT = "LIMITATIONS_PRESENT" + EVIDENCE_INCOMPLETE = "EVIDENCE_INCOMPLETE" + + # Kill switch. + KILL_SWITCH_ENGAGED = "KILL_SWITCH_ENGAGED" + KILL_SWITCH_RELEASED = "KILL_SWITCH_RELEASED" + + # Retention drill. + RETENTION_VERIFIED = "RETENTION_VERIFIED" + RETENTION_PARTIAL = "RETENTION_PARTIAL" + RETENTION_FAILED = "RETENTION_FAILED" + EXPORT_EXCLUSION_UNVERIFIED = "EXPORT_EXCLUSION_UNVERIFIED" diff --git a/src/research/analysis/operations/health.py b/src/research/analysis/operations/health.py new file mode 100644 index 0000000..f84eff4 --- /dev/null +++ b/src/research/analysis/operations/health.py @@ -0,0 +1,271 @@ +"""Operational health aggregation and alert thresholds (Issue 13). + +Health is computed from injected metadata inputs only. A monitoring outage marks +the affected window ``UNKNOWN`` and never invents health: no metric is inferred +from a missing observation, and missing coverage stays visible. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Optional + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import HealthSignalState, OperationsReasonCode +from .models import HealthSignal, OperationalHealthV1 + +if TYPE_CHECKING: + from uuid import UUID + +__all__ = [ + "HealthInputs", + "HealthThresholds", + "aggregate_health", + "health_summary", +] + +_BASE = ConfigDict(extra="forbid") + +_MONITORING_OUTAGE_REASON = "monitoring outage: health is unknown for this window" + + +class HealthThresholds(BaseModel): + """Threshold configuration for the operational signals.""" + + model_config = _BASE + + spool_depth_degraded: int = 1_000 + spool_depth_critical: int = 10_000 + spool_age_degraded_seconds: int = 3_600 + spool_age_critical_seconds: int = 86_400 + # ACK rate is a ratio; lower is worse. + upload_ack_rate_degraded: float = 0.99 + upload_ack_rate_critical: float = 0.95 + runtime_exits_degraded: int = 1 + runtime_exits_critical: int = 5 + capability_mismatch_degraded: int = 1 + capability_mismatch_critical: int = 5 + + +class HealthInputs(BaseModel): + """Metadata-only health inputs (no content fields).""" + + model_config = _BASE + + spool_depth: Optional[int] = None + spool_age_seconds: Optional[int] = None + upload_ack_rate: Optional[float] = None + proxy_start_failures: Optional[int] = None + runtime_exit_count: Optional[int] = None + capability_mismatch_count: Optional[int] = None + session_state_counts: dict[str, int] = Field(default_factory=dict) + ingestion_reject_reasons: dict[str, int] = Field(default_factory=dict) + retention_job_status: Optional[str] = None + export_job_health: Optional[str] = None + + +def _signal( + name: str, + state: HealthSignalState, + *, + threshold: Optional[str] = None, + observed: Optional[float] = None, + reason: Optional[str] = None, +) -> HealthSignal: + return HealthSignal( + name=name, + state=state, + threshold=threshold, + observed_value=None if observed is None else float(observed), + reason=reason, + ) + + +def _missing(name: str) -> HealthSignal: + return _signal( + name, + HealthSignalState.UNKNOWN, + reason="no observation for this window; health is not inferred", + ) + + +def _upper_signal( + name: str, + value: Optional[int], + *, + degraded: int, + critical: int, +) -> HealthSignal: + if value is None: + return _missing(name) + threshold = f"degraded>={degraded}, critical>={critical}" + if value >= critical: + return _signal(name, HealthSignalState.CRITICAL, threshold=threshold, observed=value) + if value >= degraded: + return _signal(name, HealthSignalState.DEGRADED, threshold=threshold, observed=value) + return _signal(name, HealthSignalState.HEALTHY, threshold=threshold, observed=value) + + +def _ack_rate_signal(value: Optional[float], thresholds: HealthThresholds) -> HealthSignal: + if value is None: + return _missing("upload_ack_rate") + threshold = ( + f"degraded<={thresholds.upload_ack_rate_degraded}, " + f"critical<={thresholds.upload_ack_rate_critical}" + ) + if value <= thresholds.upload_ack_rate_critical: + return _signal( + "upload_ack_rate", HealthSignalState.CRITICAL, threshold=threshold, observed=value + ) + if value <= thresholds.upload_ack_rate_degraded: + return _signal( + "upload_ack_rate", HealthSignalState.DEGRADED, threshold=threshold, observed=value + ) + return _signal( + "upload_ack_rate", HealthSignalState.HEALTHY, threshold=threshold, observed=value + ) + + +def _status_signal(name: str, value: Optional[str]) -> HealthSignal: + if value is None: + return _missing(name) + normalized = value.strip().upper() + if normalized in {"OK", "COMPLETE", "COMPLETED", "HEALTHY"}: + state = HealthSignalState.HEALTHY + elif normalized in {"PENDING", "RUNNING", "DEGRADED", "PARTIAL"}: + state = HealthSignalState.DEGRADED + else: + state = HealthSignalState.CRITICAL + return _signal(name, state, threshold="OK/COMPLETE healthy", observed=None) + + +def aggregate_health( + study_id: UUID, + window_start: datetime, + window_end: datetime, + inputs: HealthInputs, + *, + thresholds: Optional[HealthThresholds] = None, + monitoring_available: bool = True, + now: Optional[datetime] = None, +) -> OperationalHealthV1: + """Build an :class:`OperationalHealthV1` from metadata inputs.""" + config = thresholds or HealthThresholds() + captured = now or datetime.now(timezone.utc) + + if not monitoring_available: + # Never invent health during a monitoring outage. + signals = [ + _signal( + name, + HealthSignalState.UNKNOWN, + threshold=None, + observed=None, + reason=_MONITORING_OUTAGE_REASON, + ) + for name in ( + "spool_depth", + "spool_age_seconds", + "upload_ack_rate", + "runtime_exit_count", + "capability_mismatch_count", + "retention_job_status", + "export_job_health", + ) + ] + return OperationalHealthV1( + study_id=study_id, + window_start=window_start, + window_end=window_end, + session_state_counts={}, + ingestion_reject_reasons={}, + signals=signals, + captured_at=captured, + ) + + signals = [ + _upper_signal( + "spool_depth", + inputs.spool_depth, + degraded=config.spool_depth_degraded, + critical=config.spool_depth_critical, + ), + _upper_signal( + "spool_age_seconds", + inputs.spool_age_seconds, + degraded=config.spool_age_degraded_seconds, + critical=config.spool_age_critical_seconds, + ), + _ack_rate_signal(inputs.upload_ack_rate, config), + _upper_signal( + "runtime_exit_count", + inputs.runtime_exit_count, + degraded=config.runtime_exits_degraded, + critical=config.runtime_exits_critical, + ), + _upper_signal( + "capability_mismatch_count", + inputs.capability_mismatch_count, + degraded=config.capability_mismatch_degraded, + critical=config.capability_mismatch_critical, + ), + _status_signal("retention_job_status", inputs.retention_job_status), + _status_signal("export_job_health", inputs.export_job_health), + ] + return OperationalHealthV1( + study_id=study_id, + window_start=window_start, + window_end=window_end, + spool_depth=inputs.spool_depth, + spool_age_seconds=inputs.spool_age_seconds, + upload_ack_rate=inputs.upload_ack_rate, + proxy_start_failures=inputs.proxy_start_failures, + runtime_exit_count=inputs.runtime_exit_count, + capability_mismatch_count=inputs.capability_mismatch_count, + session_state_counts=dict(inputs.session_state_counts), + ingestion_reject_reasons=dict(inputs.ingestion_reject_reasons), + retention_job_status=inputs.retention_job_status, + export_job_health=inputs.export_job_health, + signals=signals, + captured_at=captured, + ) + + +_SEVERITY = { + HealthSignalState.HEALTHY: 0, + HealthSignalState.UNKNOWN: 1, + HealthSignalState.DEGRADED: 2, + HealthSignalState.CRITICAL: 3, +} + + +def health_summary(snapshot: OperationalHealthV1) -> dict: + """Return the worst-state summary plus explicit per-state counts.""" + overall = HealthSignalState.HEALTHY + counts = {state.value: 0 for state in HealthSignalState} + reasons = [] + for signal in snapshot.signals: + counts[signal.state.value] = counts.get(signal.state.value, 0) + 1 + if _SEVERITY[signal.state] > _SEVERITY[overall]: + overall = signal.state + if signal.state in (HealthSignalState.DEGRADED, HealthSignalState.CRITICAL): + reasons.append( + { + "code": ( + OperationsReasonCode.THRESHOLD_CRITICAL.value + if signal.state == HealthSignalState.CRITICAL + else OperationsReasonCode.THRESHOLD_DEGRADED.value + ), + "signal": signal.name, + "observed_value": signal.observed_value, + "reason": signal.reason, + } + ) + return { + "study_id": str(snapshot.study_id), + "overall": overall.value, + "counts": counts, + "reasons": reasons, + "signals": [signal.model_dump(mode="json") for signal in snapshot.signals], + } diff --git a/src/research/analysis/operations/kill_switch.py b/src/research/analysis/operations/kill_switch.py new file mode 100644 index 0000000..14e3ea8 --- /dev/null +++ b/src/research/analysis/operations/kill_switch.py @@ -0,0 +1,195 @@ +"""Operator kill switch (Issue 13). + +A kill switch is engaged at a study or enrollment scope and is +auditable (actor, reason, engaged_at). While engaged it blocks new session +bootstraps and rejects new telemetry batches; those services consult it through +an injected zero-argument predicate so they keep no operations dependency. + +The registry is pure/in-memory; the router can rebuild it from persisted rows. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import Callable, Iterable, Optional +from uuid import UUID + +from .enums import OperationsReasonCode +from .models import KillSwitchRecord, KillSwitchScope, OperationsIssue + +__all__ = [ + "KillSwitchRegistry", + "engage_kill_switch", + "is_engaged", + "kill_switch_check", + "kill_switch_issue", + "release_kill_switch", +] + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def _matches( + record: KillSwitchRecord, + *, + study_id: Optional[UUID], + enrollment_id: Optional[UUID], +) -> bool: + return record.scope.matches( + study_id=study_id, enrollment_id=enrollment_id + ) + + +class KillSwitchRegistry: + """In-memory collection of kill-switch records.""" + + def __init__(self, records: Optional[Iterable[KillSwitchRecord]] = None) -> None: + self._records: list[KillSwitchRecord] = list(records or []) + + @property + def records(self) -> list[KillSwitchRecord]: + """All records, newest engagement first.""" + return sorted(self._records, key=lambda record: record.engaged_at, reverse=True) + + def add(self, record: KillSwitchRecord) -> KillSwitchRecord: + """Register a record (e.g. rehydrated from storage).""" + self._records.append(record) + return record + + def engage( + self, + scope: KillSwitchScope, + reason: str, + *, + actor: Optional[str] = None, + now: Optional[datetime] = None, + effective_until: Optional[datetime] = None, + ) -> KillSwitchRecord: + """Engage a kill switch at ``scope``.""" + record = KillSwitchRecord( + switch_id=uuid.uuid4(), + scope=scope, + reason=reason, + actor=actor, + engaged_at=_now(now), + effective_until=effective_until, + ) + self._records.append(record) + return record + + def release( + self, switch_id: UUID, *, now: Optional[datetime] = None + ) -> Optional[KillSwitchRecord]: + """Release a switch, returning the updated record or ``None``.""" + for index, record in enumerate(self._records): + if record.switch_id == switch_id: + released = record.model_copy(update={"released_at": _now(now)}) + self._records[index] = released + return released + return None + + def active(self, *, now: Optional[datetime] = None) -> list[KillSwitchRecord]: + """Currently engaged records.""" + timestamp = _now(now) + return [record for record in self.records if record.is_engaged(timestamp)] + + def is_engaged( + self, + *, + study_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + now: Optional[datetime] = None, + ) -> bool: + """Whether any active switch covers the requested identifiers.""" + timestamp = _now(now) + return any( + record.is_engaged(timestamp) + and _matches( + record, + study_id=study_id, + enrollment_id=enrollment_id, + ) + for record in self._records + ) + + +def engage_kill_switch( + registry: KillSwitchRegistry, + scope: KillSwitchScope, + reason: str, + *, + actor: Optional[str] = None, + now: Optional[datetime] = None, + effective_until: Optional[datetime] = None, +) -> KillSwitchRecord: + """Engage a kill switch (auditable scope + reason + actor).""" + return registry.engage( + scope, reason, actor=actor, now=now, effective_until=effective_until + ) + + +def release_kill_switch( + registry: KillSwitchRegistry, switch_id: UUID, *, now: Optional[datetime] = None +) -> Optional[KillSwitchRecord]: + """Release a previously engaged switch.""" + return registry.release(switch_id, now=now) + + +def is_engaged( + registry: KillSwitchRegistry, + *, + study_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + now: Optional[datetime] = None, +) -> bool: + """Whether a kill switch blocks the requested scope.""" + return registry.is_engaged( + study_id=study_id, + enrollment_id=enrollment_id, + now=now, + ) + + +def kill_switch_issue( + registry: KillSwitchRegistry, + *, + study_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + now: Optional[datetime] = None, +) -> Optional[OperationsIssue]: + """Typed ``KILL_SWITCH_ENGAGED`` issue when the scope is blocked.""" + if is_engaged( + registry, + study_id=study_id, + enrollment_id=enrollment_id, + now=now, + ): + return OperationsIssue( + code=OperationsReasonCode.KILL_SWITCH_ENGAGED, + message="an operator kill switch is engaged for this scope", + field="kill_switch", + ) + return None + + +def kill_switch_check( + registry: KillSwitchRegistry, + *, + study_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + now: Optional[datetime] = None, +) -> Callable[[], bool]: + """Return an injected predicate for bootstrap/ingestion.""" + + def check() -> bool: + return is_engaged( + registry, + study_id=study_id, + enrollment_id=enrollment_id, + now=now, + ) + + return check diff --git a/src/research/analysis/operations/models.py b/src/research/analysis/operations/models.py new file mode 100644 index 0000000..2fd9592 --- /dev/null +++ b/src/research/analysis/operations/models.py @@ -0,0 +1,273 @@ +"""Pydantic v2 contracts for pilot operations, retention and the release gate. + +``OperationalHealthV1`` deliberately carries **no content field**: it records +metadata/health signals only (`extra="forbid"` rejects any unknown key such as a +prompt, source, or secret). ``ReleaseEvidenceV1`` is immutable; any unknown or +missing required item yields ``NO_GO``. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from research.study.protocol.enums import RetentionAction # noqa: TC001 - pydantic field type +from research.telemetry.enums import CoverageState + +from .enums import ( + EvidenceState, + HealthSignalState, + KillSwitchScopeKind, + OperationsReasonCode, + PilotClassification, + PilotOutcome, + ReleaseDecision, + RetentionVerificationStatus, +) + +_BASE = ConfigDict(extra="forbid") +_FROZEN = ConfigDict(extra="forbid", frozen=True) + +__all__ = [ + "ComponentEvidence", + "DeleteVerification", + "FaultScenario", + "HealthSignal", + "IncidentRecord", + "KillSwitchRecord", + "KillSwitchScope", + "OperationalHealthV1", + "OperationsIssue", + "PilotRunV1", + "ReleaseEvaluationResult", + "ReleaseEvidenceV1", + "ReleaseReason", + "RetentionVerification", + "ReviewerApproval", +] + + +class OperationsIssue(BaseModel): + """One typed operations rejection reason.""" + + model_config = _BASE + + code: OperationsReasonCode + message: str + field: str = "" + + +class HealthSignal(BaseModel): + """One evaluated health signal (metadata only).""" + + model_config = _FROZEN + + name: str + state: HealthSignalState = HealthSignalState.UNKNOWN + threshold: Optional[str] = None + observed_value: Optional[float] = None + reason: Optional[str] = None + + +class OperationalHealthV1(BaseModel): + """Operational health snapshot. Contains no content/prompt/secret field.""" + + model_config = _FROZEN + + study_id: UUID + window_start: datetime + window_end: datetime + spool_depth: Optional[int] = None + spool_age_seconds: Optional[int] = None + upload_ack_rate: Optional[float] = None + proxy_start_failures: Optional[int] = None + runtime_exit_count: Optional[int] = None + capability_mismatch_count: Optional[int] = None + session_state_counts: dict[str, int] = Field(default_factory=dict) + ingestion_reject_reasons: dict[str, int] = Field(default_factory=dict) + retention_job_status: Optional[str] = None + export_job_health: Optional[str] = None + signals: list[HealthSignal] = Field(default_factory=list) + captured_at: datetime + + +class ReviewerApproval(BaseModel): + """One reviewer's recorded approval (or rejection).""" + + model_config = _FROZEN + + reviewer_id: str + role: str + approved: bool + approved_at: Optional[datetime] = None + + +class ComponentEvidence(BaseModel): + """Per-component digest/state/expiry used by the gate.""" + + model_config = _FROZEN + + name: str + digest: Optional[str] = None + state: EvidenceState = EvidenceState.PRESENT + expires_at: Optional[datetime] = None + + +class ReleaseEvidenceV1(BaseModel): + """Immutable release evidence consumed by the release gate.""" + + model_config = _FROZEN + + release_id: UUID + study_id: UUID + config_digest: Optional[str] = None + config_expires_at: Optional[datetime] = None + # name -> artifact/receipt digest. A blank or UNKNOWN digest is not a pass. + component_artifacts: dict[str, str] = Field(default_factory=dict) + component_expires_at: dict[str, datetime] = Field(default_factory=dict) + # name -> PASS/FAIL/UNKNOWN (anything other than PASS is not a pass). + test_results: dict[str, str] = Field(default_factory=dict) + known_limitations: list[str] = Field(default_factory=list) + reviewer_approvals: list[ReviewerApproval] = Field(default_factory=list) + decision: Optional[ReleaseDecision] = None + recorded_at: Optional[datetime] = None + + +class ReleaseReason(BaseModel): + """One typed gate reason.""" + + model_config = _BASE + + code: OperationsReasonCode + message: str + field: str = "" + + +class ReleaseEvaluationResult(BaseModel): + """Typed release-gate decision with its reasons.""" + + model_config = _BASE + + decision: ReleaseDecision + reasons: list[ReleaseReason] = Field(default_factory=list) + config_digest: Optional[str] = None + evaluated_at: datetime + evidence: ReleaseEvidenceV1 + + +class FaultScenario(BaseModel): + """One injected fault and its observed outcome.""" + + model_config = _FROZEN + + name: str + injected: bool = True + observed_outcome: str + state: HealthSignalState = HealthSignalState.UNKNOWN + + +class IncidentRecord(BaseModel): + """One pilot incident (metadata only).""" + + model_config = _FROZEN + + incident_id: str + summary: str + severity: str + opened_at: datetime + resolved_at: Optional[datetime] = None + + +class PilotRunV1(BaseModel): + """A recorded synthetic/consented pilot run.""" + + model_config = _FROZEN + + pilot_run_id: UUID + classification: PilotClassification + cohort: list[str] = Field(default_factory=list) + window_start: datetime + window_end: datetime + fault_scenarios: list[FaultScenario] = Field(default_factory=list) + metrics_snapshot_ref: Optional[str] = None + coverage_snapshot_ref: Optional[str] = None + incidents: list[IncidentRecord] = Field(default_factory=list) + outcome: PilotOutcome + recorded_at: datetime + + +class KillSwitchScope(BaseModel): + """The scope a kill switch applies to.""" + + model_config = _FROZEN + + kind: KillSwitchScopeKind + scope_id: UUID + + def matches( + self, + *, + study_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + ) -> bool: + """Whether this scope covers the requested identifiers.""" + if self.kind == KillSwitchScopeKind.STUDY: + return study_id is not None and study_id == self.scope_id + return enrollment_id is not None and enrollment_id == self.scope_id + + +class KillSwitchRecord(BaseModel): + """One auditable kill-switch engagement (or release).""" + + model_config = _BASE + + switch_id: UUID + scope: KillSwitchScope + reason: str + actor: Optional[str] = None + engaged_at: datetime + effective_until: Optional[datetime] = None + released_at: Optional[datetime] = None + + def is_engaged(self, now: datetime) -> bool: + """Whether the switch is currently engaged.""" + if self.released_at is not None: + return False + if self.effective_until is not None and now >= self.effective_until: + return False + return True + + +class DeleteVerification(BaseModel): + """Per-record deletion verification evidence (metadata only).""" + + model_config = _FROZEN + + record_id: str + expected_state: str + observed_state: str + verified: bool + + +class RetentionVerification(BaseModel): + """Result of a deletion-retention drill for one enrollment.""" + + model_config = _BASE + + verification_id: UUID + enrollment_id: UUID + action: RetentionAction + status: RetentionVerificationStatus + evidence_digest: str + verified_at: datetime + deleted_count: int = 0 + remaining_count: int = 0 + expected_remaining_count: int = 0 + export_exclusion_verified: Optional[bool] = None + coverage: CoverageState = CoverageState.UNKNOWN + close_out_allowed: bool = False + reasons: list[OperationsIssue] = Field(default_factory=list) + record_checks: list[DeleteVerification] = Field(default_factory=list) diff --git a/src/research/analysis/operations/release_gate.py b/src/research/analysis/operations/release_gate.py new file mode 100644 index 0000000..6f55b02 --- /dev/null +++ b/src/research/analysis/operations/release_gate.py @@ -0,0 +1,185 @@ +"""Release gate evaluator (Issue 13). + +``evaluate_release`` is a pure function over immutable evidence plus the +required component/evidence lists. Any unknown or missing required item yields +``NO_GO``; an expired component receipt or fixed study configuration yields +``EXPIRED``; a GO with recorded limitations is explicitly ``GO_WITH_LIMITS``. +A material artifact/configuration change invalidates an old GO. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Iterable, Optional + +from .enums import OperationsReasonCode, ReleaseDecision +from .models import ( + ReleaseEvaluationResult, + ReleaseEvidenceV1, + ReleaseReason, +) + +__all__ = [ + "EXPIRED_REASONS", + "HARD_REASONS", + "NON_BLOCKING_REASONS", + "evaluate_release", + "record_decision", +] + +HARD_REASONS = frozenset( + { + OperationsReasonCode.REQUIRED_COMPONENT_MISSING, + OperationsReasonCode.REQUIRED_EVIDENCE_MISSING, + OperationsReasonCode.MATERIAL_CHANGE, + OperationsReasonCode.EVIDENCE_INCOMPLETE, + } +) + +EXPIRED_REASONS = frozenset( + { + OperationsReasonCode.COMPONENT_RECEIPT_EXPIRED, + OperationsReasonCode.CONFIG_EXPIRED, + } +) + +NON_BLOCKING_REASONS = frozenset({OperationsReasonCode.LIMITATIONS_PRESENT}) + +_PASS = "PASS" +_UNKNOWN_DIGESTS = frozenset({"", "UNKNOWN", "MISSING", "LATEST"}) + + +def _reason( + code: OperationsReasonCode, message: str, field: str = "" +) -> ReleaseReason: + return ReleaseReason(code=code, message=message, field=field) + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def evaluate_release( + evidence: ReleaseEvidenceV1, + *, + required_components: Iterable[str], + required_evidence: Iterable[str], + now: Optional[datetime] = None, + current_component_digests: Optional[dict[str, str]] = None, + current_config_digest: Optional[str] = None, +) -> ReleaseEvaluationResult: + """Evaluate release evidence and return the typed gate decision.""" + timestamp = _now(now) + reasons: list[ReleaseReason] = [] + + for name in required_components: + digest = evidence.component_artifacts.get(name) + if digest is None: + reasons.append( + _reason( + OperationsReasonCode.REQUIRED_COMPONENT_MISSING, + f"required component {name!r} has no recorded digest", + name, + ) + ) + elif digest.strip().upper() in _UNKNOWN_DIGESTS: + reasons.append( + _reason( + OperationsReasonCode.REQUIRED_COMPONENT_MISSING, + f"required component {name!r} digest is unknown", + name, + ) + ) + + expires_at = evidence.component_expires_at.get(name) + if expires_at is not None and expires_at < timestamp: + reasons.append( + _reason( + OperationsReasonCode.COMPONENT_RECEIPT_EXPIRED, + f"component {name!r} receipt expired at {expires_at.isoformat()}", + name, + ) + ) + + if current_component_digests is not None: + current = current_component_digests.get(name) + if current is not None and digest is not None and current != digest: + reasons.append( + _reason( + OperationsReasonCode.MATERIAL_CHANGE, + f"component {name!r} digest changed since the recorded evidence", + name, + ) + ) + + for name in required_evidence: + state = evidence.test_results.get(name) + if state is None or state.strip().upper() != _PASS: + reasons.append( + _reason( + OperationsReasonCode.REQUIRED_EVIDENCE_MISSING, + f"required evidence {name!r} is missing or not PASS", + name, + ) + ) + + if evidence.config_expires_at is not None and evidence.config_expires_at < timestamp: + reasons.append( + _reason( + OperationsReasonCode.CONFIG_EXPIRED, + "the fixed study configuration expired at " + f"{evidence.config_expires_at.isoformat()}", + "study_id", + ) + ) + + if ( + current_config_digest is not None + and evidence.config_digest is not None + and current_config_digest != evidence.config_digest + ): + reasons.append( + _reason( + OperationsReasonCode.MATERIAL_CHANGE, + "the study configuration digest changed since the recorded evidence", + "study_id", + ) + ) + + codes = {reason.code for reason in reasons} + if codes & EXPIRED_REASONS: + decision = ReleaseDecision.EXPIRED + elif codes & HARD_REASONS: + decision = ReleaseDecision.NO_GO + elif evidence.known_limitations: + reasons.append( + _reason( + OperationsReasonCode.LIMITATIONS_PRESENT, + "release is approved with recorded limitations: " + + "; ".join(evidence.known_limitations), + "known_limitations", + ) + ) + decision = ReleaseDecision.GO_WITH_LIMITS + else: + decision = ReleaseDecision.GO + + return ReleaseEvaluationResult( + decision=decision, + reasons=reasons, + config_digest=evidence.config_digest, + evaluated_at=timestamp, + evidence=evidence, + ) + + +def record_decision( + evidence: ReleaseEvidenceV1, + decision: ReleaseDecision, + *, + recorded_at: Optional[datetime] = None, +) -> ReleaseEvidenceV1: + """Return an immutable copy of ``evidence`` with the gate decision recorded.""" + return evidence.model_copy( + update={"decision": decision, "recorded_at": recorded_at or _now(None)} + ) diff --git a/src/research/analysis/operations/retention.py b/src/research/analysis/operations/retention.py new file mode 100644 index 0000000..53f8c09 --- /dev/null +++ b/src/research/analysis/operations/retention.py @@ -0,0 +1,180 @@ +"""Deletion-retention drill verification (Issue 13). + +Given an enrollment, its pre-deletion records, the observed post-deletion +records, and (optionally) the exported record ids, verify that the declared +retention action was actually executed and that exports exclude deleted data. + +A partial failure does **not** silently close out: the verification status is +``PARTIAL``/``FAILED``, ``close_out_allowed`` is ``False``, and ledger evidence +(the expected deletion ledger plus an evidence digest) is retained. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Optional, Sequence +from uuid import UUID + +from research.canonical import canonical_hash +from research.participants import identity as participants_identity +from research.telemetry.enums import CoverageState + +from .enums import OperationsReasonCode, RetentionVerificationStatus +from .models import DeleteVerification, OperationsIssue, RetentionVerification + +if TYPE_CHECKING: + from research.participants.models import Enrollment, PseudonymousRecord + from research.study.protocol.enums import RetentionAction + +__all__ = ["verify_deletion"] + +_LINKABLE = ("account_id", "email", "session_token") + + +def _issue( + code: OperationsReasonCode, message: str, field: str = "" +) -> OperationsIssue: + return OperationsIssue(code=code, message=message, field=field) + + +def _has_linkable(record: PseudonymousRecord) -> bool: + return any(getattr(record, name) is not None for name in _LINKABLE) + + +def verify_deletion( + enrollment: Enrollment, + *, + action: RetentionAction, + before_records: Sequence[PseudonymousRecord], + after_records: Optional[Sequence[PseudonymousRecord]] = None, + export_record_ids: Optional[Sequence[str]] = None, + verified_at: Optional[datetime] = None, + verification_id: Optional[UUID] = None, +) -> RetentionVerification: + """Verify a declared retention action was executed for one enrollment.""" + timestamp = verified_at or datetime.now(timezone.utc) + expected = participants_identity.apply_retention( + enrollment, list(before_records), action, now=timestamp + ) + observed = ( + list(after_records) if after_records is not None else list(expected.records) + ) + + expected_by_id = {record.record_id: record for record in expected.records} + observed_by_id = {record.record_id: record for record in observed} + + before_ids = {record.record_id for record in before_records} + expected_ids = set(expected_by_id) + deleted_ids = before_ids - expected_ids + observed_ids = set(observed_by_id) + + checks: list[DeleteVerification] = [] + missing_present = sorted(expected_ids - observed_ids) + extra_present = sorted(observed_ids - expected_ids) + linkable_left = sorted( + record_id + for record_id, record in observed_by_id.items() + if _has_linkable(record) + ) + + for record_id in sorted(before_ids | observed_ids): + should_exist = record_id in expected_ids + does_exist = record_id in observed_by_id + checks.append( + DeleteVerification( + record_id=record_id, + expected_state="present" if should_exist else "deleted", + observed_state="present" if does_exist else "deleted", + verified=should_exist == does_exist, + ) + ) + + reasons: list[OperationsIssue] = [] + if missing_present: + status = RetentionVerificationStatus.FAILED + reasons.append( + _issue( + OperationsReasonCode.RETENTION_FAILED, + f"{len(missing_present)} retained record(s) were missing after deletion", + "after_records", + ) + ) + elif extra_present or linkable_left: + status = RetentionVerificationStatus.PARTIAL + if extra_present: + reasons.append( + _issue( + OperationsReasonCode.RETENTION_PARTIAL, + f"{len(extra_present)} record(s) were not deleted", + "after_records", + ) + ) + if linkable_left: + reasons.append( + _issue( + OperationsReasonCode.RETENTION_PARTIAL, + f"{len(linkable_left)} record(s) still expose account-linkable fields", + "after_records", + ) + ) + else: + status = RetentionVerificationStatus.VERIFIED + reasons.append( + _issue( + OperationsReasonCode.RETENTION_VERIFIED, + "declared retention action executed for every record", + ) + ) + + export_exclusion_verified: Optional[bool] = None + if export_record_ids is not None: + exported = set(export_record_ids) + export_exclusion_verified = not (deleted_ids & exported) + if not export_exclusion_verified: + status = RetentionVerificationStatus.PARTIAL + reasons.append( + _issue( + OperationsReasonCode.EXPORT_EXCLUSION_UNVERIFIED, + "an export still references deleted record(s)", + "export_record_ids", + ) + ) + + coverage = ( + CoverageState.AVAILABLE + if status == RetentionVerificationStatus.VERIFIED + else CoverageState.PARTIAL + ) + close_out_allowed = ( + status == RetentionVerificationStatus.VERIFIED + and export_exclusion_verified is not False + ) + + evidence_digest = canonical_hash( + { + "action": action.value, + "enrollment_id": str(enrollment.enrollment_id), + "expected_ledger": expected.ledger.model_dump(mode="json"), + "observed_ids": sorted(observed_ids), + "deleted_ids": sorted(deleted_ids), + "status": status.value, + } + ) + + return RetentionVerification( + verification_id=verification_id or uuid.uuid4(), + enrollment_id=enrollment.enrollment_id, + action=action, + status=status, + evidence_digest=evidence_digest, + verified_at=timestamp, + deleted_count=len(deleted_ids), + remaining_count=len(observed_ids), + expected_remaining_count=len(expected_ids), + export_exclusion_verified=export_exclusion_verified, + coverage=coverage, + close_out_allowed=close_out_allowed, + reasons=reasons, + record_checks=checks, + ) diff --git a/src/research/analysis/operations/store.py b/src/research/analysis/operations/store.py new file mode 100644 index 0000000..52c6455 --- /dev/null +++ b/src/research/analysis/operations/store.py @@ -0,0 +1,426 @@ +"""Persistence helpers for pilot operations, release gate and kill switch. + +Every helper takes a caller-supplied SQLAlchemy ``Session`` and returns domain +models (never raw ORM rows) so the domain layer stays free of DB/framework +imports. Kill-switch engagements, health snapshots, release evidence and pilot +runs are all consolidated into the single generic ``research_record`` table. + +Deletion-drill *verification* is a pure function +(:func:`research.analysis.operations.retention.verify_deletion`); the single +retention-evidence writer is the identity package's ``insert_deletion_ledger``, +which writes ``research_record`` (``kind = RETENTION_EVIDENCE``) and merges onto +``research_retention_job.evidence_json``. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Callable, Optional, Sequence + +from sqlalchemy import and_, or_, select + +from database.research_schemas import ( + RECORD_KIND_HEALTH, + RECORD_KIND_KILL_SWITCH, + RECORD_KIND_PILOT_RUN, + RECORD_KIND_RELEASE_EVIDENCE, + ResearchRecord, +) +from research.analysis.operations.kill_switch import KillSwitchRegistry + +from .enums import KillSwitchScopeKind +from .models import ( + KillSwitchRecord, + OperationalHealthV1, + PilotRunV1, + ReleaseEvidenceV1, +) + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + from .enums import PilotClassification + +__all__ = [ + "create_health", + "create_pilot_run", + "create_release_evidence", + "db_kill_switch_check", + "engage_kill_switch", + "get_health", + "get_kill_switch", + "get_pilot_run", + "get_release_evidence", + "is_kill_switch_engaged", + "kill_switch_records", + "list_health", + "list_pilot_runs", + "list_release_evidence", + "release_kill_switch", + "row_to_kill_switch", + "set_release_decision", +] + + +def _record( + *, + record_id: uuid.UUID, + kind: str, + occurred_at: datetime, + payload_json: dict, + scope_type: Optional[str] = None, + scope_id: Optional[uuid.UUID] = None, + study_id: Optional[uuid.UUID] = None, + actor: Optional[str] = None, +) -> ResearchRecord: + return ResearchRecord( + record_id=record_id, + kind=kind, + scope_type=scope_type, + scope_id=scope_id, + study_id=study_id, + actor=actor, + occurred_at=occurred_at, + payload_json=payload_json, + ) + + +# --------------------------------------------------------------------------- +# Operational health +# --------------------------------------------------------------------------- + + +def create_health(session: Session, snapshot: OperationalHealthV1) -> ResearchRecord: + """Insert one health snapshot (the full model is stored as JSON).""" + row = _record( + record_id=uuid.uuid4(), + kind=RECORD_KIND_HEALTH, + occurred_at=snapshot.captured_at, + payload_json=snapshot.model_dump(mode="json"), + scope_type="study", + scope_id=snapshot.study_id, + study_id=snapshot.study_id, + ) + session.add(row) + session.commit() + session.refresh(row) + return row + + +def _row_to_health(row: ResearchRecord) -> OperationalHealthV1: + return OperationalHealthV1.model_validate(row.payload_json) + + +def get_health( + session: Session, health_id: uuid.UUID +) -> Optional[OperationalHealthV1]: + """Fetch one health snapshot by id, or ``None``.""" + row = session.get(ResearchRecord, health_id) + if row is None or row.kind != RECORD_KIND_HEALTH: + return None + return _row_to_health(row) + + +def list_health( + session: Session, study_id: uuid.UUID +) -> Sequence[OperationalHealthV1]: + """List a study's health snapshots, oldest-first.""" + statement = ( + select(ResearchRecord) + .where( + ResearchRecord.study_id == study_id, + ResearchRecord.kind == RECORD_KIND_HEALTH, + ) + .order_by(ResearchRecord.occurred_at.asc()) + ) + return [_row_to_health(row) for row in session.execute(statement).scalars().all()] + + +# --------------------------------------------------------------------------- +# Release evidence +# --------------------------------------------------------------------------- + + +def create_release_evidence( + session: Session, evidence: ReleaseEvidenceV1 +) -> ReleaseEvidenceV1: + """Insert one immutable release-evidence record.""" + row = _record( + record_id=evidence.release_id, + kind=RECORD_KIND_RELEASE_EVIDENCE, + occurred_at=evidence.recorded_at or datetime.now(), + payload_json=evidence.model_dump(mode="json"), + scope_type="study", + scope_id=evidence.study_id, + ) + session.add(row) + session.commit() + session.refresh(row) + return evidence + + +def _row_to_release_evidence(row: ResearchRecord) -> ReleaseEvidenceV1: + return ReleaseEvidenceV1.model_validate(row.payload_json) + + +def get_release_evidence( + session: Session, release_id: uuid.UUID +) -> Optional[ReleaseEvidenceV1]: + """Fetch one release-evidence record by id, or ``None``.""" + row = session.get(ResearchRecord, release_id) + if row is None or row.kind != RECORD_KIND_RELEASE_EVIDENCE: + return None + return _row_to_release_evidence(row) + + +def list_release_evidence( + session: Session, study_id: uuid.UUID +) -> Sequence[ReleaseEvidenceV1]: + """List release evidence for a study, oldest-first.""" + statement = ( + select(ResearchRecord) + .where( + ResearchRecord.kind == RECORD_KIND_RELEASE_EVIDENCE, + ResearchRecord.payload_json["study_id"].astext + == str(study_id), + ) + .order_by(ResearchRecord.occurred_at.asc()) + ) + return [ + _row_to_release_evidence(row) + for row in session.execute(statement).scalars().all() + ] + + +def set_release_decision( + session: Session, evidence: ReleaseEvidenceV1 +) -> ResearchRecord: + """Record a gate decision on an existing evidence row. + + Only the gate output (the decision fields inside the stored JSON) is + written; the immutable inputs are never changed. + """ + row = session.get(ResearchRecord, evidence.release_id) + if row is None or row.kind != RECORD_KIND_RELEASE_EVIDENCE: + raise ValueError(f"release evidence {evidence.release_id} not found") + row.payload_json = evidence.model_dump(mode="json") + session.add(row) + session.commit() + session.refresh(row) + return row + + +# --------------------------------------------------------------------------- +# Pilot runs +# --------------------------------------------------------------------------- + + +def create_pilot_run(session: Session, run: PilotRunV1) -> PilotRunV1: + """Insert one recorded pilot run.""" + row = _record( + record_id=run.pilot_run_id, + kind=RECORD_KIND_PILOT_RUN, + occurred_at=run.recorded_at, + payload_json=run.model_dump(mode="json"), + ) + session.add(row) + session.commit() + session.refresh(row) + return run + + +def _row_to_pilot_run(row: ResearchRecord) -> PilotRunV1: + return PilotRunV1.model_validate(row.payload_json) + + +def get_pilot_run( + session: Session, pilot_run_id: uuid.UUID +) -> Optional[PilotRunV1]: + """Fetch one pilot run by id, or ``None``.""" + row = session.get(ResearchRecord, pilot_run_id) + if row is None or row.kind != RECORD_KIND_PILOT_RUN: + return None + return _row_to_pilot_run(row) + + +def list_pilot_runs( + session: Session, classification: Optional[PilotClassification] = None +) -> Sequence[PilotRunV1]: + """List pilot runs, optionally filtered by classification.""" + statement = select(ResearchRecord).where( + ResearchRecord.kind == RECORD_KIND_PILOT_RUN + ) + if classification is not None: + statement = statement.where( + ResearchRecord.payload_json["classification"].astext + == classification.value + ) + statement = statement.order_by(ResearchRecord.occurred_at.asc()) + return [ + _row_to_pilot_run(row) + for row in session.execute(statement).scalars().all() + ] + + +# --------------------------------------------------------------------------- +# Kill switch +# --------------------------------------------------------------------------- + + +def engage_kill_switch( + session: Session, record: KillSwitchRecord +) -> KillSwitchRecord: + """Persist a kill-switch engagement.""" + row = _record( + record_id=record.switch_id, + kind=RECORD_KIND_KILL_SWITCH, + occurred_at=record.engaged_at, + payload_json=record.model_dump(mode="json"), + scope_type=record.scope.kind.value, + scope_id=record.scope.scope_id, + actor=record.actor, + ) + session.add(row) + session.commit() + session.refresh(row) + return record + + +def release_kill_switch( + session: Session, switch_id: uuid.UUID, *, + released_at: Optional[datetime] = None, +) -> Optional[KillSwitchRecord]: + """Mark a switch released, returning the record or ``None``.""" + row = session.get(ResearchRecord, switch_id) + if row is None or row.kind != RECORD_KIND_KILL_SWITCH: + return None + payload = dict(row.payload_json or {}) + payload["released_at"] = (released_at or datetime.now()).isoformat() + row.payload_json = payload + session.add(row) + session.commit() + session.refresh(row) + return row_to_kill_switch(row) + + +def get_kill_switch( + session: Session, switch_id: uuid.UUID +) -> Optional[KillSwitchRecord]: + """Fetch a kill-switch record by id, or ``None``.""" + row = session.get(ResearchRecord, switch_id) + if row is None or row.kind != RECORD_KIND_KILL_SWITCH: + return None + return row_to_kill_switch(row) + + +def row_to_kill_switch(row: ResearchRecord) -> KillSwitchRecord: + """Rehydrate a kill-switch record row into a domain record.""" + return KillSwitchRecord.model_validate(row.payload_json) + + +def kill_switch_records(session: Session) -> Sequence[KillSwitchRecord]: + """List all kill-switch records, newest engagement first.""" + statement = ( + select(ResearchRecord) + .where(ResearchRecord.kind == RECORD_KIND_KILL_SWITCH) + .order_by(ResearchRecord.occurred_at.desc()) + ) + return [ + row_to_kill_switch(row) for row in session.execute(statement).scalars().all() + ] + + +def latest_study_kill_switch( + session: Session, study_id: uuid.UUID +) -> Optional[KillSwitchRecord]: + """Return the newest persisted study-scoped switch, if one exists.""" + statement = ( + select(ResearchRecord) + .where( + ResearchRecord.kind == RECORD_KIND_KILL_SWITCH, + ResearchRecord.scope_type == "STUDY", + ResearchRecord.scope_id == study_id, + ) + .order_by(ResearchRecord.occurred_at.desc()) + .limit(1) + ) + row = session.execute(statement).scalars().first() + return row_to_kill_switch(row) if row is not None else None + + +def is_kill_switch_engaged( + session: Session, + *, + study_id: Optional[uuid.UUID] = None, + enrollment_id: Optional[uuid.UUID] = None, + now: Optional[datetime] = None, +) -> bool: + """Whether a persisted ``KILL_SWITCH`` record covers the requested scope. + + This is the DB-backed counterpart of the in-memory + :class:`~research.analysis.operations.kill_switch.KillSwitchRegistry`: it reads + only the records whose scope can possibly match the requested identifiers + (``STUDY`` for the study, ``ENROLLMENT`` for the enrollment) and evaluates the + same scope matching. A record whose scope kind does not cover the requested + identifiers does not block, so the SQL filter is exactly equivalent to the + previous load-everything-then-filter behaviour without the O(all rows) read. + """ + scope_conditions = [] + if study_id is not None: + scope_conditions.append( + and_( + ResearchRecord.scope_type == KillSwitchScopeKind.STUDY.value, + ResearchRecord.scope_id == study_id, + ) + ) + if enrollment_id is not None: + scope_conditions.append( + and_( + ResearchRecord.scope_type == KillSwitchScopeKind.ENROLLMENT.value, + ResearchRecord.scope_id == enrollment_id, + ) + ) + if not scope_conditions: + return False + statement = ( + select(ResearchRecord) + .where( + ResearchRecord.kind == RECORD_KIND_KILL_SWITCH, + or_(*scope_conditions), + ) + .order_by(ResearchRecord.occurred_at.desc()) + ) + registry = KillSwitchRegistry( + [row_to_kill_switch(row) for row in session.execute(statement).scalars().all()] + ) + return registry.is_engaged( + study_id=study_id, + enrollment_id=enrollment_id, + now=now or datetime.now(timezone.utc), + ) + + +def db_kill_switch_check( + session: Session, + *, + study_id: Optional[uuid.UUID] = None, + enrollment_id: Optional[uuid.UUID] = None, + now: Optional[datetime] = None, +) -> Callable[[], bool]: + """Return a zero-argument kill-switch predicate backed by the database. + + The predicate re-reads the persisted records on every call (the services + invoke it at the point of decision), so an engagement committed between the + request starting and the check taking effect is observed. + """ + + def check() -> bool: + return is_kill_switch_engaged( + session, + study_id=study_id, + enrollment_id=enrollment_id, + now=now, + ) + + return check diff --git a/src/research/analysis/read_models/__init__.py b/src/research/analysis/read_models/__init__.py new file mode 100644 index 0000000..d5a570f --- /dev/null +++ b/src/research/analysis/read_models/__init__.py @@ -0,0 +1,86 @@ +"""Researcher read models and scoped RBAC (Issue 12). + +Read models carry study/profile identity, coverage/derivation version, and the +explicit population definition. Unavailable data is ``null`` + a coverage state, +never zero. + +Public surface: + +* :mod:`research.analysis.read_models.enums` - researcher roles and typed reason codes. +* :mod:`research.analysis.read_models.models` - ``StudySummaryV1``, + ``EnrollmentCoverageV1``, ``TelemetryCoverageV1``, + ``DerivedMetricV1``. +* :mod:`research.analysis.read_models.rbac` - per-study authorization (privacy operator + access is separate from telemetry read). +* :mod:`research.analysis.read_models.service` - pure builders. +* :mod:`research.analysis.read_models.store` - Session-supplied queries. + +The core package never imports ``App``, FastAPI, or a session factory. +""" + +from .enums import ReadModelReasonCode, ResearcherRole +from .models import ( + COVERAGE_VERSION, + DERIVATION_VERSION, + DerivedMetricV1, + EnrollmentCoverageV1, + FamilyCoverage, + ParticipantAssignmentV1, + ParticipantCoverageRowV1, + ParticipantEventCoverageV1, + ParticipantSessionCoverageV1, + ReadModelIssue, + StudyParticipantCoverageV1, + StudySummaryV1, + TelemetryCoverageV1, +) +from .rbac import ( + ReadModelAuthorizationError, + ResearcherGrant, + can_read_private_mapping, + can_read_study, + can_read_telemetry, + require_study_access, + require_telemetry_access, + role_for_study, +) +from .service import ( + build_derived_metrics, + build_enrollment_coverage, + build_participant_coverage, + build_study_summary, + build_telemetry_coverage, + population_definition, +) + +__all__ = [ + "COVERAGE_VERSION", + "DERIVATION_VERSION", + "DerivedMetricV1", + "EnrollmentCoverageV1", + "FamilyCoverage", + "ParticipantAssignmentV1", + "ParticipantCoverageRowV1", + "ParticipantEventCoverageV1", + "ParticipantSessionCoverageV1", + "ReadModelAuthorizationError", + "ReadModelIssue", + "ReadModelReasonCode", + "ResearcherGrant", + "ResearcherRole", + "StudyParticipantCoverageV1", + "StudySummaryV1", + "TelemetryCoverageV1", + "build_derived_metrics", + "build_enrollment_coverage", + "build_participant_coverage", + "build_revision_summary", + "build_telemetry_coverage", + "can_read_private_mapping", + "can_read_study", + "can_read_telemetry", + "population_definition", + "require_study_access", + "require_telemetry_access", + "role_for_study", +] diff --git a/src/research/analysis/read_models/dashboard.py b/src/research/analysis/read_models/dashboard.py new file mode 100644 index 0000000..3979a24 --- /dev/null +++ b/src/research/analysis/read_models/dashboard.py @@ -0,0 +1,591 @@ +"""Canonical, owner-scoped agent analytics read path (phase 06 cutover). + +One query path serves both the aggregate overview and the per-run detail, reading +the canonical ``research_event`` authority instead of the legacy ``agent_event`` +store. Agent tasks join their canonical events by the explicit phase-05 binding +(``research_event.agent_run_id = agent_task.external_run_id``) — never by an ACP +session id or a time window. + +Ownership is enforced before any join: a non-administrator only ever aggregates +tasks they own (``agent_task.owner_user_id``); an administrator may optionally +scope to one user. A legacy task with no canonical events renders explicit +unavailable/unknown values (``None`` for averages and counts that are genuinely +unknown), never fabricated zeros. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta +from typing import Any, Optional + +from sqlalchemy import text + +_TIME_WINDOWS = {"7d": 7, "30d": 30, "90d": 90} + +#: The canonical payload marker the relay/self-report adapters attach to each +#: observation, so a relay ``model_call`` is never confused with an ACP +#: ``agent.message.completed`` for the same run (source ownership). +_LEGACY_KIND = "e.envelope_json -> 'payload' ->> 'legacy_kind'" +_LATENCY = "(e.envelope_json -> 'metrics' ->> 'latency_ms')::float" +_USAGE = "(e.envelope_json -> 'metrics' ->> 'usage_tokens')::bigint" +_UPSTREAM = "e.envelope_json -> 'payload' ->> 'upstream_status'" +_PROMPT_TOKENS = "(e.envelope_json -> 'metrics' -> 'counts' ->> 'prompt_tokens')::bigint" +_COMPLETION_TOKENS = ( + "(e.envelope_json -> 'metrics' -> 'counts' ->> 'completion_tokens')::bigint" +) +_TOOL_NAME = "e.envelope_json -> 'payload' ->> 'tool_name'" +_TOOL_SCHEMA_BYTES = "(e.envelope_json -> 'payload' ->> 'tool_schema_bytes')::bigint" +_CONTEXT_BYTES = ( + "(e.envelope_json -> 'payload' ->> 'context_window_size_bytes')::bigint" +) +_TOOL_RESULT_LENGTH = ( + "(e.envelope_json -> 'payload' ->> 'tool_result_length')::bigint" +) +_DECISION = "e.envelope_json -> 'payload' ->> 'decision'" +#: A permission decision the built-in agent's user made. As with the model and +#: tool counts, only the agent's own reports are read (for a proxied run the ACP +#: proxy observes the same round-trips). Decisions no one made are left out, by +#: the rule of ``study_analytics.metrics.relay_decision_made``: scope ``policy`` +#: (auto-approval, a suggestion-only refusal) or ``session_cached`` (an earlier +#: "allow for this session"), and ``unavailable`` (the request reached no one). +_USER_DECISION = ( + f"({_LEGACY_KIND} = 'permission_decided' AND {_DECISION} IS NOT NULL " + f"AND {_DECISION} <> 'unavailable' " + "AND COALESCE(e.envelope_json -> 'payload' ->> 'decision_scope', '') " + "NOT IN ('policy', 'session_cached'))" +) + +_EVENT_JOIN = ( + "FROM research_event e " + "JOIN agent_task t ON t.external_run_id = e.agent_run_id" +) + + +def _seconds_window(time_window: str) -> Optional[int]: + days = _TIME_WINDOWS.get(time_window) + return days + + +def _local_naive(value: datetime) -> datetime: + """``value`` as the naive local time ``agent_task.created_at`` is written in.""" + if value.tzinfo is None: + return value + return value.astimezone().replace(tzinfo=None) + + +def _filters( + current_user, + *, + user_id: Optional[str], + framework: Optional[str], + model: Optional[str], + profile: Optional[str], + start_time: datetime, + end_time: datetime, +) -> tuple[str, dict[str, Any]]: + """Owner scope first, then the optional filters (never before ownership).""" + params: dict[str, Any] = {"start_time": start_time, "end_time": end_time} + conditions = ["t.created_at BETWEEN :start_time AND :end_time"] + is_admin = bool(getattr(current_user, "is_admin", False)) + if is_admin and user_id: + params["user_id"] = user_id + conditions.append("CAST(t.owner_user_id AS TEXT) = :user_id") + elif not is_admin: + params["user_id"] = str(current_user.user_id) + conditions.append("t.owner_user_id = CAST(:user_id AS UUID)") + if framework: + params["framework"] = framework + conditions.append("t.framework_version = :framework") + if model: + params["model"] = model + conditions.append("t.model = :model") + if profile: + params["profile"] = profile + conditions.append("t.agent_profile = :profile") + return " AND ".join(conditions), params + + +def agent_overview( + db: Any, + current_user, + *, + time_window: str, + framework: Optional[str] = None, + model: Optional[str] = None, + profile: Optional[str] = None, + user_id: Optional[str] = None, + now: Optional[datetime] = None, +) -> dict[str, Any]: + """Aggregate canonical agent events for the owner's tasks.""" + days = _seconds_window(time_window) + if days is None: + raise ValueError("time_window must be 7d, 30d, or 90d") + # ``agent_task.created_at`` is stored as a naive local timestamp + # (``default=datetime.now``); an aware UTC bound is shifted by the server's + # UTC offset on the way in, which hid the most recent tasks off-UTC. + end_time = _local_naive(now) if now is not None else datetime.now() + start_time = end_time - timedelta(days=days) + where, params = _filters( + current_user, + user_id=user_id, + framework=framework, + model=model, + profile=profile, + start_time=start_time, + end_time=end_time, + ) + + task = db.execute( + text( + f""" + SELECT + COUNT(*) AS total_tasks, + COUNT(*) FILTER (WHERE t.status = 'done') AS completed_tasks, + COUNT(*) FILTER (WHERE t.status = 'failed') AS failed_tasks, + COUNT(*) FILTER (WHERE t.status IN ('pending', 'running')) AS open_tasks, + COALESCE(AVG(t.total_steps), 0) AS avg_steps, + COALESCE(AVG(EXTRACT(EPOCH FROM (t.completed_at - t.started_at)) * 1000) + FILTER (WHERE t.started_at IS NOT NULL AND t.completed_at IS NOT NULL), 0) + AS avg_task_duration_ms + FROM agent_task t + WHERE {where} + """ + ), + params, + ).one() + + events = db.execute( + text( + f""" + SELECT + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'model_call') AS model_calls, + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'tool_call') AS tool_calls, + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'tool_failed') AS tool_failures, + COUNT(*) FILTER (WHERE e.event_type IN ('agent.error', 'system.proxy.error', + 'system.agent.crashed', 'unknown_source_event') + AND {_LEGACY_KIND} IS NULL) AS observed_failures, + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'model_call' + AND ({_UPSTREAM})::int >= 200 AND ({_UPSTREAM})::int < 400) + AS successful_model_calls, + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'model_call' + AND ({_UPSTREAM})::int = 429) AS rate_limit_retries, + SUM({_USAGE}) FILTER (WHERE {_LEGACY_KIND} = 'model_call') AS event_tokens, + SUM({_PROMPT_TOKENS}) FILTER (WHERE {_LEGACY_KIND} = 'model_call') + AS provider_input_tokens, + SUM({_COMPLETION_TOKENS}) FILTER (WHERE {_LEGACY_KIND} = 'model_call') + AS model_output_tokens, + SUM({_TOOL_SCHEMA_BYTES}) AS tool_schema_bytes, + SUM({_CONTEXT_BYTES}) AS conversation_context_bytes, + SUM({_TOOL_RESULT_LENGTH}) AS tool_result_bytes, + COUNT(*) FILTER (WHERE {_USER_DECISION}) AS permission_decisions, + COUNT(*) FILTER (WHERE {_USER_DECISION} AND {_DECISION} = 'accepted') + AS permission_accepted, + AVG({_LATENCY}) FILTER (WHERE {_LEGACY_KIND} = 'model_call') + AS avg_model_latency_ms, + PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY {_LATENCY}) + FILTER (WHERE {_LEGACY_KIND} = 'model_call' AND {_LATENCY} IS NOT NULL) + AS p95_model_latency_ms + {_EVENT_JOIN} + WHERE {where} + """ + ), + params, + ).one() + + profile_rows = db.execute( + text( + f""" + SELECT + COALESCE(t.agent_profile, 'unknown') AS profile_name, + COALESCE(t.framework_version, 'unknown') AS framework_version, + COALESCE(t.model, 'unknown') AS model, + COUNT(*) AS tasks, + COUNT(*) FILTER (WHERE t.status = 'done') AS completed, + COUNT(*) FILTER (WHERE t.status = 'failed') AS failed, + COALESCE(AVG(t.total_steps), 0) AS avg_steps + FROM agent_task t + WHERE {where} + GROUP BY 1, 2, 3 + ORDER BY tasks DESC, profile_name + """ + ), + params, + ).fetchall() + + tool_rows = db.execute( + text( + f""" + SELECT + COALESCE({_TOOL_NAME}, 'unknown') AS tool_name, + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'tool_call') AS calls, + COUNT(*) FILTER (WHERE {_LEGACY_KIND} = 'tool_failed') AS failures, + COALESCE(AVG({_LATENCY}), 0) AS avg_latency_ms + {_EVENT_JOIN} + WHERE {where} AND {_LEGACY_KIND} IN ('tool_call', 'tool_failed') + GROUP BY 1 + ORDER BY calls DESC, tool_name + LIMIT 20 + """ + ), + params, + ).fetchall() + + run_rows = db.execute( + text( + f""" + SELECT + t.task_id, t.agent_profile, t.framework_version, t.model, t.status, + t.total_steps, t.created_at, t.started_at, t.completed_at, + COUNT(e.event_id) FILTER (WHERE {_LEGACY_KIND} = 'model_call') AS model_calls, + COUNT(e.event_id) FILTER (WHERE {_LEGACY_KIND} = 'tool_call') AS tool_calls, + COUNT(e.event_id) FILTER (WHERE {_LEGACY_KIND} = 'tool_failed') AS failures + FROM agent_task t + LEFT JOIN research_event e ON e.agent_run_id = t.external_run_id + WHERE {where} + GROUP BY t.task_id + ORDER BY t.created_at DESC + LIMIT 25 + """ + ), + params, + ).fetchall() + + trend_rows = db.execute( + text( + f""" + SELECT + DATE_TRUNC('day', t.created_at) AS day, + COUNT(*) AS runs, + COUNT(*) FILTER (WHERE t.status = 'done') AS completed, + COUNT(*) FILTER (WHERE t.status = 'failed') AS failed, + COALESCE(AVG(EXTRACT(EPOCH FROM (t.completed_at - t.started_at)) * 1000) + FILTER (WHERE t.started_at IS NOT NULL AND t.completed_at IS NOT NULL), 0) + AS avg_duration_ms + FROM agent_task t + WHERE {where} + GROUP BY 1 + ORDER BY 1 + """ + ), + params, + ).fetchall() + + latency_rows = db.execute( + text( + f""" + SELECT + CASE + WHEN {_LATENCY} < 250 THEN '<250ms' + WHEN {_LATENCY} < 500 THEN '250-499ms' + WHEN {_LATENCY} < 1000 THEN '500-999ms' + WHEN {_LATENCY} < 2000 THEN '1-2s' + ELSE '2s+' + END AS bucket, + CASE + WHEN {_LATENCY} < 250 THEN 1 + WHEN {_LATENCY} < 500 THEN 2 + WHEN {_LATENCY} < 1000 THEN 3 + WHEN {_LATENCY} < 2000 THEN 4 + ELSE 5 + END AS bucket_order, + COUNT(*) AS calls + {_EVENT_JOIN} + WHERE {where} AND {_LEGACY_KIND} = 'model_call' AND {_LATENCY} IS NOT NULL + GROUP BY 1, 2 + ORDER BY bucket_order + """ + ), + params, + ).fetchall() + + event_type_rows = db.execute( + text( + f""" + SELECT + COALESCE({_LEGACY_KIND}, e.event_type) AS event_type, + COUNT(*) AS events + {_EVENT_JOIN} + WHERE {where} + GROUP BY 1 + ORDER BY events DESC, event_type + """ + ), + params, + ).fetchall() + + error_rows = db.execute( + text( + f""" + SELECT + CASE + WHEN {_LEGACY_KIND} = 'tool_failed' THEN 'tool_failed' + WHEN ({_UPSTREAM})::int >= 400 THEN 'upstream_' || ({_UPSTREAM}) + ELSE e.event_type + END AS reason, + COUNT(*) AS events + {_EVENT_JOIN} + WHERE {where} + AND ( + {_LEGACY_KIND} = 'tool_failed' + OR ({_UPSTREAM})::int >= 400 + OR e.event_type IN ('agent.error', 'system.proxy.error', 'system.agent.crashed') + ) + GROUP BY 1 + ORDER BY events DESC, reason + """ + ), + params, + ).fetchall() + + def num(value, integer: bool = False): + if value is None: + return None + return int(value) if integer else float(value) + + total_tasks = num(task.total_tasks, integer=True) or 0 + tool_failures = num(events.tool_failures, integer=True) or 0 + observed_failures = num(events.observed_failures, integer=True) or 0 + return { + "filters": { + "time_window": time_window, + "framework": framework, + "model": model, + "profile": profile, + }, + "summary": { + "total_tasks": total_tasks, + "completed_tasks": num(task.completed_tasks, integer=True) or 0, + "failed_tasks": num(task.failed_tasks, integer=True) or 0, + "open_tasks": num(task.open_tasks, integer=True) or 0, + "completion_rate": (num(task.completed_tasks, integer=True) or 0) + / max(total_tasks, 1), + "avg_steps": num(task.avg_steps), + "avg_task_duration_ms": num(task.avg_task_duration_ms), + "model_calls": num(events.model_calls, integer=True) or 0, + "tool_calls": num(events.tool_calls, integer=True) or 0, + "failures": tool_failures + observed_failures, + # Canonical latencies/tokens are null when no canonical model-call + # fact exists (a legacy run with no canonical events), never zero. + "avg_model_latency_ms": num(events.avg_model_latency_ms), + "p95_model_latency_ms": num(events.p95_model_latency_ms), + "provider_input_tokens": num(events.provider_input_tokens, integer=True), + "model_output_tokens": num(events.model_output_tokens, integer=True), + "event_tokens": num(events.event_tokens, integer=True), + "successful_model_calls": num(events.successful_model_calls, integer=True) or 0, + "rate_limit_retries": num(events.rate_limit_retries, integer=True) or 0, + "tool_schema_tokens_estimated": ( + (num(events.tool_schema_bytes, integer=True) or 0) // 4 + if events.tool_schema_bytes is not None + else None + ), + "conversation_tokens_estimated": ( + (num(events.conversation_context_bytes, integer=True) or 0) // 4 + if events.conversation_context_bytes is not None + else None + ), + "tool_result_tokens_estimated": ( + (num(events.tool_result_bytes, integer=True) or 0) // 4 + if events.tool_result_bytes is not None + else None + ), + # Edit acceptance is computed from recorded permission decisions: + # accepted tool executions over all decisions a user was asked for + # (see ``_USER_DECISION``). With no such decision both stay null and + # the dashboard renders unavailable rather than a fabricated ratio. + "edit_acceptance_rate": ( + (num(events.permission_accepted, integer=True) or 0) + / (num(events.permission_decisions, integer=True) or 0) + if (num(events.permission_decisions, integer=True) or 0) > 0 + else None + ), + "total_edits": ( + num(events.permission_decisions, integer=True) or None + ), + }, + "profiles": [ + { + "profile_name": row.profile_name, + "framework_version": row.framework_version, + "model": row.model, + "tasks": num(row.tasks, integer=True), + "completed": num(row.completed, integer=True), + "failed": num(row.failed, integer=True), + "completion_rate": (num(row.completed, integer=True) or 0) + / max(num(row.tasks, integer=True) or 0, 1), + "avg_steps": num(row.avg_steps), + } + for row in profile_rows + ], + "tools": [ + { + "tool_name": row.tool_name, + "calls": num(row.calls, integer=True), + "failures": num(row.failures, integer=True), + "avg_latency_ms": num(row.avg_latency_ms), + } + for row in tool_rows + ], + "recent_runs": [ + { + "task_id": str(row.task_id), + "profile_name": row.agent_profile, + "framework_version": row.framework_version, + "model": row.model, + "status": row.status, + "steps": num(row.total_steps, integer=True), + "created_at": _iso(row.created_at), + "started_at": _iso(row.started_at), + "completed_at": _iso(row.completed_at), + "model_calls": num(row.model_calls, integer=True), + "tool_calls": num(row.tool_calls, integer=True), + "failures": num(row.failures, integer=True), + } + for row in run_rows + ], + "trend": [ + { + "day": _iso(row.day), + "runs": num(row.runs, integer=True), + "completed": num(row.completed, integer=True), + "failed": num(row.failed, integer=True), + "avg_duration_ms": num(row.avg_duration_ms), + } + for row in trend_rows + ], + "latency_distribution": [ + {"bucket": row.bucket, "calls": num(row.calls, integer=True)} + for row in latency_rows + ], + "event_types": [ + {"event_type": row.event_type, "events": num(row.events, integer=True)} + for row in event_type_rows + ], + "errors": [ + {"reason": row.reason, "events": num(row.events, integer=True)} + for row in error_rows + ], + } + + +def agent_run_detail( + db: Any, + current_user, + *, + task_id: str, +) -> Optional[dict[str, Any]]: + """Return one owner-scoped run and its canonical events, or ``None``.""" + params: dict[str, Any] = {"task_id": task_id} + owner_filter = "" + if not getattr(current_user, "is_admin", False): + params["user_id"] = str(current_user.user_id) + owner_filter = "AND CAST(t.owner_user_id AS TEXT) = :user_id" + + task = db.execute( + text( + f""" + SELECT t.task_id, t.agent_profile, t.framework_version, t.model, + t.status, t.total_steps, t.created_at, t.started_at, t.completed_at, + t.external_run_id + FROM agent_task t + WHERE CAST(t.task_id AS TEXT) = :task_id {owner_filter} + """ + ), + params, + ).one_or_none() + if task is None: + return None + + event_rows = db.execute( + text( + f""" + SELECT + COALESCE({_LEGACY_KIND}, e.event_type) AS event_type, + e.envelope_json -> 'provenance' ->> 'source' AS source, + {_LATENCY} AS latency_ms, + {_UPSTREAM} AS upstream_status, + {_TOOL_NAME} AS tool_name, + e.envelope_json -> 'payload' ->> 'model' AS model, + {_USAGE} AS total_tokens, + e.envelope_json -> 'metrics' -> 'counts' ->> 'prompt_tokens' AS prompt_tokens, + e.envelope_json -> 'metrics' -> 'counts' ->> 'completion_tokens' AS completion_tokens, + e.envelope_json -> 'payload' ->> 'finish_reason' AS finish_reason, + e.envelope_json -> 'metrics' -> 'counts' ->> 'step_index' AS step_index, + e.envelope_json -> 'correlations' ->> 'trace_id' AS trace_id, + e.envelope_json -> 'correlations' ->> 'span_id' AS span_id, + e.envelope_json -> 'correlations' ->> 'parent_span_id' AS parent_span_id, + e.envelope_json -> 'correlations' ->> 'model_call_id' AS model_call_id, + e.envelope_json -> 'correlations' ->> 'message_id' AS message_id, + COALESCE(e.envelope_json -> 'payload' ->> 'request_id', + e.envelope_json -> 'correlations' ->> 'correlation_id') + AS request_id, + e.occurred_at AS occurred_at, + e.envelope_json -> 'correlations' ->> 'tool_call_id' AS tool_call_id, + e.event_type AS canonical_event_type + FROM research_event e + WHERE e.agent_run_id = :run_id + ORDER BY e.occurred_at, e.emitter_sequence + """ + ), + {"run_id": task.external_run_id}, + ).fetchall() + + events = [] + for position, row in enumerate(event_rows, start=1): + events.append( + { + "event_index": position, + "event_type": row.event_type, + "source": row.source, + "latency_ms": _num(row.latency_ms), + "created_at": None, + "occurred_at": _iso(row.occurred_at), + "trace_id": row.trace_id, + "span_id": row.span_id, + "parent_span_id": row.parent_span_id, + "model_call_id": row.model_call_id, + "message_id": row.message_id, + "request_id": row.request_id, + "model": row.model, + "tool_name": row.tool_name, + "prompt_tokens": _num(row.prompt_tokens), + "completion_tokens": _num(row.completion_tokens), + "total_tokens": _num(row.total_tokens), + "finish_reason": row.finish_reason, + "upstream_status": _num(row.upstream_status), + "step_index": _num(row.step_index), + "detail": row.canonical_event_type + if row.event_type != row.canonical_event_type + else None, + } + ) + + return { + "run": { + "task_id": str(task.task_id), + "profile_name": task.agent_profile, + "framework_version": task.framework_version, + "model": task.model, + "status": task.status, + "steps": int(task.total_steps or 0), + "created_at": _iso(task.created_at), + "started_at": _iso(task.started_at), + "completed_at": _iso(task.completed_at), + }, + "events": events, + } + + +def _iso(value): + return value.isoformat() if value is not None else None + + +def _num(value): + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + try: + return float(value) + except (TypeError, ValueError): + return None diff --git a/src/research/analysis/read_models/enums.py b/src/research/analysis/read_models/enums.py new file mode 100644 index 0000000..8f268f3 --- /dev/null +++ b/src/research/analysis/read_models/enums.py @@ -0,0 +1,38 @@ +"""Closed vocabularies for researcher read models (Issue 12). + +Roles and reason codes are additive-only contracts: a new role or reason is a +new member, never a renamed one. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = ["ReadModelReasonCode", "ResearcherRole"] + + +class ResearcherRole(str, Enum): + """Researcher access role granted per study. + + ``PRIVACY_OPERATOR`` is deliberately separate: it may read the private + account-to-participant mapping but **never** telemetry content/read models. + """ + + OWNER = "OWNER" + ANALYST = "ANALYST" + VIEWER = "VIEWER" + PRIVACY_OPERATOR = "PRIVACY_OPERATOR" + + +class ReadModelReasonCode(str, Enum): + """Stable machine-readable reason a read-model request was rejected.""" + + OK = "OK" + FORBIDDEN_STUDY = "FORBIDDEN_STUDY" + TELEMETRY_FORBIDDEN = "TELEMETRY_FORBIDDEN" + NO_ROLE = "NO_ROLE" + NO_REVISION = "NO_REVISION" + REVISION_MISMATCH = "REVISION_MISMATCH" + INVALID_POPULATION = "INVALID_POPULATION" + RETENTION_BLOCKED = "RETENTION_BLOCKED" + WRITER_UNAVAILABLE = "WRITER_UNAVAILABLE" diff --git a/src/research/analysis/read_models/models.py b/src/research/analysis/read_models/models.py new file mode 100644 index 0000000..f59dbec --- /dev/null +++ b/src/research/analysis/read_models/models.py @@ -0,0 +1,146 @@ +"""Study/profile-scoped researcher read models.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from research.telemetry.enums import CoverageState +from research.telemetry.projections.models import COVERAGE_VERSION, DERIVATION_VERSION, FamilyCoverage + +_BASE = ConfigDict(extra="forbid") + + +class StudySummaryV1(BaseModel): + model_config = _BASE + + study_id: UUID + name: str + research_status: Optional[str] = None + profile_count: int = 0 + profile_ids: list[UUID] = Field(default_factory=list) + config_digest: Optional[str] = None + population: str = "study" + + +class EnrollmentCoverageV1(BaseModel): + model_config = _BASE + + study_id: UUID + coverage_version: str = COVERAGE_VERSION + population: str + total_enrollments: int = 0 + status_counts: dict[str, int] = Field(default_factory=dict) + coverage: CoverageState = CoverageState.UNKNOWN + coverage_reason: Optional[str] = None + + +class TelemetryCoverageV1(BaseModel): + model_config = _BASE + + study_id: UUID + coverage_version: str = COVERAGE_VERSION + population: str + families: list[FamilyCoverage] = Field(default_factory=list) + denominators: dict[str, int] = Field(default_factory=dict) + computed_at: Optional[datetime] = None + + +class ParticipantAssignmentV1(BaseModel): + """The frozen assignment facts a study owner may inspect (no snapshot).""" + + model_config = _BASE + + assignment_id: UUID + agent_profile_id: UUID + strategy: str + randomization_epoch: int + profile_digest: str + status: str + assigned_at: datetime + + +class ParticipantSessionCoverageV1(BaseModel): + """Session counts and liveness for one enrolled participant.""" + + model_config = _BASE + + total: int = 0 + active: int = 0 + terminal: int = 0 + last_activity_at: Optional[datetime] = None + last_heartbeat_at: Optional[datetime] = None + + +class ParticipantEventCoverageV1(BaseModel): + """Canonical event counts for one enrolled participant.""" + + model_config = _BASE + + total: int = 0 + by_event_type: dict[str, int] = Field(default_factory=dict) + by_source: dict[str, int] = Field(default_factory=dict) + last_occurred_at: Optional[datetime] = None + + +class ParticipantCoverageRowV1(BaseModel): + """One study-local participant with assignment and coverage facts. + + Keyed by ``enrollment_id``/``participant_code`` only: account, participant + and login identity are never part of this projection. + """ + + model_config = _BASE + + enrollment_id: UUID + participant_code: str + status: str + enrolled_at: datetime + updated_at: datetime + assignment: Optional[ParticipantAssignmentV1] = None + sessions: ParticipantSessionCoverageV1 = Field( + default_factory=ParticipantSessionCoverageV1 + ) + events: ParticipantEventCoverageV1 = Field( + default_factory=ParticipantEventCoverageV1 + ) + + +class StudyParticipantCoverageV1(BaseModel): + """Study-owner-scoped per-participant read model (no cross-study rows).""" + + model_config = _BASE + + study_id: UUID + coverage_version: str = COVERAGE_VERSION + population: str = "enrolled" + participant_count: int = 0 + participants: list[ParticipantCoverageRowV1] = Field(default_factory=list) + coverage: CoverageState = CoverageState.UNKNOWN + coverage_reason: Optional[str] = None + + +class DerivedMetricV1(BaseModel): + model_config = _BASE + + study_id: UUID + derivation_version: str = DERIVATION_VERSION + population: str + metric_id: str + numerator: Optional[int] = None + denominator: Optional[int] = None + value: Optional[float] = None + coverage_state: CoverageState = CoverageState.UNKNOWN + coverage_predicate: str + source_event_count: int = 0 + + +class ReadModelIssue(BaseModel): + model_config = _BASE + + code: str + message: str + field: str = "" diff --git a/src/research/analysis/read_models/rbac.py b/src/research/analysis/read_models/rbac.py new file mode 100644 index 0000000..44bafdb --- /dev/null +++ b/src/research/analysis/read_models/rbac.py @@ -0,0 +1,128 @@ +"""Scoped researcher RBAC (Issue 12). + +Authorization is per study. A researcher with a role on one study is rejected for +another study: scope is never partially widened by role alone. The privacy +operator capability is separate from telemetry read access. +""" + +from __future__ import annotations + +from typing import Iterable, Optional +from uuid import UUID # noqa: TC003 - pydantic resolves ResearcherGrant.study_id at runtime + +from pydantic import BaseModel, ConfigDict + +from .enums import ReadModelReasonCode, ResearcherRole + +__all__ = [ + "ReadModelAuthorizationError", + "ResearcherGrant", + "can_read_private_mapping", + "can_read_study", + "can_read_telemetry", + "require_study_access", + "require_telemetry_access", + "role_for_study", +] + +_BASE = ConfigDict(extra="forbid") + +_TELEMETRY_ROLES = frozenset( + {ResearcherRole.OWNER, ResearcherRole.ANALYST, ResearcherRole.VIEWER} +) +_PRIVATE_MAPPING_ROLES = frozenset({ResearcherRole.PRIVACY_OPERATOR}) + + +class ResearcherGrant(BaseModel): + """One researcher's role grant on one study.""" + + model_config = _BASE + + researcher_id: str + study_id: UUID + role: ResearcherRole + + +class ReadModelAuthorizationError(PermissionError): + """Typed authorization failure for a read-model/export request.""" + + def __init__(self, code: ReadModelReasonCode, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +def role_for_study( + grants: Iterable[ResearcherGrant], + researcher_id: str, + study_id: UUID, +) -> Optional[ResearcherRole]: + """Return the researcher's role on ``study_id``, or ``None``.""" + for grant in grants: + if grant.researcher_id == researcher_id and grant.study_id == study_id: + return grant.role + return None + + +def can_read_study( + role: Optional[ResearcherRole], + study_id: UUID, + *, + permitted_study_ids: Iterable[UUID], +) -> bool: + """Whether ``role`` may read ``study_id``. + + Requires BOTH an assigned role and an explicit study permission. A role on a + different study never grants access here. + """ + if role is None: + return False + permitted = set(permitted_study_ids) + return study_id in permitted + + +def can_read_telemetry(role: Optional[ResearcherRole]) -> bool: + """Whether ``role`` may read telemetry read models/exports. + + Privacy operators are excluded: they work with the private mapping, never + telemetry content. + """ + return role in _TELEMETRY_ROLES + + +def can_read_private_mapping(role: Optional[ResearcherRole]) -> bool: + """Whether ``role`` may read the private account-to-participant mapping.""" + return role in _PRIVATE_MAPPING_ROLES + + +def require_study_access( + grants: Iterable[ResearcherGrant], + researcher_id: str, + study_id: UUID, +) -> ResearcherRole: + """Return the role or raise a typed authorization error.""" + grants = list(grants) + role = role_for_study(grants, researcher_id, study_id) + permitted = [grant.study_id for grant in grants if grant.researcher_id == researcher_id] + if not can_read_study(role, study_id, permitted_study_ids=permitted): + # Distinguish "this researcher has no grants at all" from "this + # researcher has grants elsewhere but not on this study". + code = ( + ReadModelReasonCode.NO_ROLE + if not permitted + else ReadModelReasonCode.FORBIDDEN_STUDY + ) + raise ReadModelAuthorizationError( + code, f"researcher is not authorized for study {study_id}" + ) + assert role is not None + return role + + +def require_telemetry_access(role: ResearcherRole) -> None: + """Raise when ``role`` may not read telemetry read models.""" + if not can_read_telemetry(role): + raise ReadModelAuthorizationError( + ReadModelReasonCode.TELEMETRY_FORBIDDEN, + f"role {role.value} may not read telemetry read models", + ) diff --git a/src/research/analysis/read_models/service.py b/src/research/analysis/read_models/service.py new file mode 100644 index 0000000..8564df0 --- /dev/null +++ b/src/research/analysis/read_models/service.py @@ -0,0 +1,263 @@ +"""Pure study/profile-scoped read-model builders.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Iterable, Optional, Sequence + +from research.telemetry.enums import CoverageState +from research.telemetry.projections.models import COVERAGE_VERSION +from research.telemetry.projections.service import coverage_by_family, derive_event_count_metric, derive_usage_metric + +from .models import ( + DerivedMetricV1, + EnrollmentCoverageV1, + ParticipantAssignmentV1, + ParticipantCoverageRowV1, + ParticipantEventCoverageV1, + ParticipantSessionCoverageV1, + StudyParticipantCoverageV1, + StudySummaryV1, + TelemetryCoverageV1, +) + +if TYPE_CHECKING: + from datetime import datetime + from uuid import UUID + +__all__ = [ + "build_derived_metrics", + "build_enrollment_coverage", + "build_participant_coverage", + "build_study_summary", + "build_telemetry_coverage", + "population_definition", +] + +_POPULATIONS = { + "study": "all records bound to this study", + "all_events": "all stored canonical events for the study", + "enrolled": "all enrolled participants for the study", +} + + +def population_definition(population: str) -> str: + return _POPULATIONS.get(population, f"custom population {population!r}") + + +def _status_value(value: Any) -> str: + return value.value if hasattr(value, "value") else str(value) + + +def build_study_summary(study: Any, profiles: Iterable[Any] = ()) -> StudySummaryV1: + profile_list = list(profiles) + return StudySummaryV1( + study_id=study.study_id, + name=study.name, + research_status=getattr(study, "research_status", None), + profile_count=len(profile_list), + profile_ids=[profile.profile_id for profile in profile_list], + config_digest=getattr(study, "research_config_digest", None), + ) + + +def build_enrollment_coverage( + enrollments: Iterable[Any], + *, + study_id: UUID, + population: str = "enrolled", +) -> EnrollmentCoverageV1: + status_counts: dict[str, int] = {} + total = 0 + for enrollment in enrollments: + total += 1 + status = _status_value(enrollment.status) + status_counts[status] = status_counts.get(status, 0) + 1 + return EnrollmentCoverageV1( + study_id=study_id, + population=population, + total_enrollments=total, + status_counts=status_counts, + coverage=CoverageState.AVAILABLE if total else CoverageState.UNAVAILABLE, + coverage_reason=None if total else "no enrollments in population", + ) + + +def build_telemetry_coverage( + records: Iterable[Any], + *, + study_id: UUID, + population: str = "all_events", +) -> TelemetryCoverageV1: + projection = coverage_by_family(list(records), study_id, population=population) + return TelemetryCoverageV1( + study_id=study_id, + coverage_version=projection.coverage_version, + population=projection.population, + families=projection.families, + denominators=projection.denominators, + computed_at=projection.computed_at, + ) + + +def build_derived_metrics( + records: Sequence[Any], + *, + study_id: UUID, + population: str = "all_events", +) -> list[DerivedMetricV1]: + metrics = [ + derive_event_count_metric(list(records), population=population), + derive_usage_metric(list(records), population=population), + ] + return [ + DerivedMetricV1( + study_id=study_id, + derivation_version=metric.derivation_version, + population=population, + metric_id=metric.metric_id, + numerator=metric.numerator, + denominator=metric.denominator, + value=metric.value, + coverage_state=metric.coverage_state, + coverage_predicate=metric.coverage_predicate, + source_event_count=metric.source_event_count, + ) + for metric in metrics + ] + + +def _later( + current: Optional[datetime], candidate: Optional[datetime] +) -> Optional[datetime]: + """Return the later of two optional timestamps (``None``-safe).""" + if candidate is None: + return current + if current is None or candidate > current: + return candidate + return current + + +def build_participant_coverage( + enrollments: Iterable[Any], + assignments: Iterable[Any], + sessions: Iterable[Any], + events: Iterable[Any], + *, + study_id: UUID, + population: str = "enrolled", +) -> StudyParticipantCoverageV1: + """Build the study-owner-scoped per-participant coverage read model. + + Every input is already study-scoped by the caller, so rows can never mix + studies. Rows are keyed by the study-local ``enrollment_id`` / + ``participant_code`` only; login identity is never joined. An event whose + ``enrollment_id`` is missing (e.g. the enrollment row was removed) is not + attributed to a participant row rather than being merged into another. + """ + assignment_by_enrollment: dict[Any, Any] = {} + for assignment in assignments: + assignment_by_enrollment.setdefault(assignment.enrollment_id, assignment) + + session_stats: dict[Any, dict[str, Any]] = {} + for research_session in sessions: + stats = session_stats.setdefault( + research_session.enrollment_id, + { + "total": 0, + "active": 0, + "terminal": 0, + "last_activity_at": None, + "last_heartbeat_at": None, + }, + ) + stats["total"] += 1 + if research_session.state.is_terminal: + stats["terminal"] += 1 + else: + stats["active"] += 1 + stats["last_activity_at"] = _later( + stats["last_activity_at"], research_session.last_activity_at + ) + stats["last_heartbeat_at"] = _later( + stats["last_heartbeat_at"], research_session.last_heartbeat_at + ) + + event_stats: dict[Any, dict[str, Any]] = {} + for record in events: + enrollment_id = record.enrollment_id + if enrollment_id is None: + continue + stats = event_stats.setdefault( + enrollment_id, + { + "total": 0, + "by_event_type": {}, + "by_source": {}, + "last_occurred_at": None, + }, + ) + stats["total"] += 1 + stats["by_event_type"][record.event_type] = ( + stats["by_event_type"].get(record.event_type, 0) + 1 + ) + stats["by_source"][record.source] = stats["by_source"].get(record.source, 0) + 1 + stats["last_occurred_at"] = _later( + stats["last_occurred_at"], record.occurred_at + ) + + participants: list[ParticipantCoverageRowV1] = [] + for enrollment in sorted( + enrollments, key=lambda item: (item.enrolled_at, item.participant_code) + ): + assignment = assignment_by_enrollment.get(enrollment.enrollment_id) + session_summary = session_stats.get(enrollment.enrollment_id) or {} + event_summary = event_stats.get(enrollment.enrollment_id) or {} + participants.append( + ParticipantCoverageRowV1( + enrollment_id=enrollment.enrollment_id, + participant_code=enrollment.participant_code, + status=_status_value(enrollment.status), + enrolled_at=enrollment.enrolled_at, + updated_at=enrollment.updated_at, + assignment=( + ParticipantAssignmentV1( + assignment_id=assignment.assignment_id, + agent_profile_id=assignment.agent_profile_id, + strategy=assignment.strategy, + randomization_epoch=assignment.randomization_epoch, + profile_digest=assignment.profile_digest, + status=assignment.status, + assigned_at=assignment.assigned_at, + ) + if assignment is not None + else None + ), + sessions=ParticipantSessionCoverageV1( + total=session_summary.get("total", 0), + active=session_summary.get("active", 0), + terminal=session_summary.get("terminal", 0), + last_activity_at=session_summary.get("last_activity_at"), + last_heartbeat_at=session_summary.get("last_heartbeat_at"), + ), + events=ParticipantEventCoverageV1( + total=event_summary.get("total", 0), + by_event_type=dict( + sorted(event_summary.get("by_event_type", {}).items()) + ), + by_source=dict( + sorted(event_summary.get("by_source", {}).items()) + ), + last_occurred_at=event_summary.get("last_occurred_at"), + ), + ) + ) + + total = len(participants) + return StudyParticipantCoverageV1( + study_id=study_id, + population=population, + participant_count=total, + participants=participants, + coverage=CoverageState.AVAILABLE if total else CoverageState.UNAVAILABLE, + coverage_reason=None if total else "no enrollments in population", + ) diff --git a/src/research/analysis/read_models/store.py b/src/research/analysis/read_models/store.py new file mode 100644 index 0000000..0cfb448 --- /dev/null +++ b/src/research/analysis/read_models/store.py @@ -0,0 +1,156 @@ +"""Session-supplied read queries for the researcher control plane (Issue 12). + +These helpers are thin wrappers over the canonical stores: they return domain +records (never raw ORM/JSON) and never join login identity into a read model. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional, Sequence + +from sqlalchemy import select + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + from research.participants.models import Enrollment + from research.runtime.assignment.models import AssignmentV1 + from research.runtime.sessions.models import AgentRunV1, ResearchSessionV1 + from research.telemetry.ingestion.models import ResearchEventRecord + +from database.research_schemas import ( + ResearchAgentRun, + ResearchEnrollment, + ResearchEvent, + StudyAssignment, +) +from database.research_schemas import ( + ResearchSessionV1 as ResearchSessionRow, +) +from research.participants import identity as identity_store +from research.runtime.assignment import store as assignment_store +from research.runtime.sessions import store as session_store +from research.telemetry.ingestion import store as ingestion_store + +__all__ = [ + "StudyCoverageInputs", + "list_agent_runs", + "list_assignments", + "list_coverage_inputs", + "list_enrollments", + "list_events", + "list_sessions", +] + + +@dataclass(frozen=True) +class StudyCoverageInputs: + """One study's coverage inputs as domain records (no login identity).""" + + enrollments: Sequence[Enrollment] + assignments: Sequence[AssignmentV1] + sessions: Sequence[ResearchSessionV1] + events: Sequence[ResearchEventRecord] + + +def list_enrollments(session: Session, study_id: uuid.UUID) -> Sequence[Enrollment]: + """List a study's enrollments as domain records (no login identity).""" + statement = select(ResearchEnrollment).where(ResearchEnrollment.study_id == study_id) + return [ + identity_store.row_to_enrollment(row) + for row in session.execute(statement).scalars().all() + ] + + +def list_assignments(session: Session, study_id: uuid.UUID) -> Sequence[AssignmentV1]: + """List a study's assignments as domain records (joined via enrollment).""" + statement = ( + select(StudyAssignment) + .join( + ResearchEnrollment, + StudyAssignment.enrollment_id == ResearchEnrollment.enrollment_id, + ) + .where(ResearchEnrollment.study_id == study_id) + ) + return [ + assignment_store.row_to_assignment(row) + for row in session.execute(statement).scalars().all() + ] + + +def list_sessions( + session: Session, study_id: uuid.UUID +) -> Sequence[ResearchSessionV1]: + """List a study's research sessions as domain records.""" + statement = ( + select(ResearchSessionRow) + .join(ResearchEnrollment, ResearchSessionRow.enrollment_id == ResearchEnrollment.enrollment_id) + .where(ResearchEnrollment.study_id == study_id) + .order_by(ResearchSessionRow.created_at.asc()) + ) + return [ + session_store.row_to_session(row) + for row in session.execute(statement).scalars().all() + ] + + +def list_agent_runs( + session: Session, research_session_id: uuid.UUID +) -> Sequence[AgentRunV1]: + """List the agent runs of one session as domain records.""" + statement = ( + select(ResearchAgentRun) + .where(ResearchAgentRun.research_session_id == research_session_id) + .order_by(ResearchAgentRun.started_at.asc()) + ) + return [ + session_store.row_to_agent_run(row) + for row in session.execute(statement).scalars().all() + ] + + +def list_events( + session: Session, + study_id: uuid.UUID, + *, + research_session_ids: Optional[Sequence[uuid.UUID]] = None, +) -> Sequence[ResearchEventRecord]: + """List a study's canonical events as records (sorted deterministically). + + Retention tombstones (``retention_state == "DELETED"``) are excluded so a + post-withdrawal export never reads deleted data. + """ + statement = select(ResearchEvent).where( + ResearchEvent.study_id == study_id, + ResearchEvent.retention_state != "DELETED", + ) + if research_session_ids is not None: + statement = statement.where( + ResearchEvent.research_session_id.in_(list(research_session_ids)) + ) + statement = statement.order_by( + ResearchEvent.research_session_id.asc(), + ResearchEvent.emitter_id.asc(), + ResearchEvent.emitter_sequence.asc(), + ) + return [ + ingestion_store.row_to_record(row) + for row in session.execute(statement).scalars().all() + ] + + +def list_coverage_inputs(session: Session, study_id: uuid.UUID) -> StudyCoverageInputs: + """Load every study-scoped input the participant coverage model needs. + + One call composes the existing study-scoped helpers so the per-table row + mapping stays in one place. The events query reuses ``list_events`` and + therefore excludes retention tombstones (``retention_state == "DELETED"``). + """ + return StudyCoverageInputs( + enrollments=list_enrollments(session, study_id), + assignments=list_assignments(session, study_id), + sessions=list_sessions(session, study_id), + events=list_events(session, study_id), + ) diff --git a/src/research/analysis/study_analytics/__init__.py b/src/research/analysis/study_analytics/__init__.py new file mode 100644 index 0000000..926abed --- /dev/null +++ b/src/research/analysis/study_analytics/__init__.py @@ -0,0 +1,52 @@ +"""Researcher study analytics: participant table, dashboard and arm comparison. + +Issue 03 of run ``2026-09-24-research-ui-overhaul``. Read-only, study-scoped +read models over the canonical telemetry store: + +* :mod:`research.analysis.study_analytics.models` - plain row types; +* :mod:`research.analysis.study_analytics.store` - read-only SQL (metadata JSONB + paths only, retention tombstones excluded, no login identity); +* :mod:`research.analysis.study_analytics.metrics` - pure participant-level + metrics and the three response builders. + +The participant (enrollment) is the randomisation unit: arms are compared on +per-participant values, following the agent-trace analyses of SWE-chat and the +work built on it. The package never imports ``App``, FastAPI or a session +factory. +""" + +from .metrics import ( + FALLBACK_MAX_CONTEXT_TOKENS, + METRIC_KEYS, + analyze_participant, + build_participant_detail, + build_participants, + build_study_summary, +) +from .models import ( + ArmRow, + AssignmentRow, + DailyEventCount, + DateWindow, + EnrollmentRow, + EventRow, + SessionRow, + StudyFrame, +) + +__all__ = [ + "ArmRow", + "AssignmentRow", + "DailyEventCount", + "DateWindow", + "EnrollmentRow", + "EventRow", + "FALLBACK_MAX_CONTEXT_TOKENS", + "METRIC_KEYS", + "SessionRow", + "StudyFrame", + "analyze_participant", + "build_participant_detail", + "build_participants", + "build_study_summary", +] diff --git a/src/research/analysis/study_analytics/metrics.py b/src/research/analysis/study_analytics/metrics.py new file mode 100644 index 0000000..ed24a63 --- /dev/null +++ b/src/research/analysis/study_analytics/metrics.py @@ -0,0 +1,2350 @@ +"""Pure participant-level study analytics over plain rows (issue 03). + +Everything here is a pure function of :mod:`research.analysis.study_analytics.models` +rows: no database, no FastAPI, no clock (``now`` is always a parameter). The +participant (enrollment) is the randomisation unit, so every arm comparison +aggregates within a participant first and then summarises the per-participant +values. + +Event semantics (verified against the producers on 2026-09-24): + +* ``agent.message.started`` is emitted by the shared ACP normalizer for a + ``session/prompt`` request (the user prompt, a turn start: no + ``payload.message_kind`` and no lifecycle state) *and* for every streamed + ``agent_message_chunk`` / ``agent_thought_chunk`` (``payload.message_kind`` = + ``assistant``/``thought``, lifecycle ``started``). Only the former is a prompt. +* ``agent.message.completed`` observed by the ACP proxy is the prompt response + (``payload.stop_reason``, ``metrics.usage_tokens``). With source ``relay`` it + is one provider model call (``payload.legacy_kind = model_call``; emitter + ``relay`` for ``/api/agent/inference``, ``self-report`` for the managed + runtime's ``agent.model.completed``, ``proxy`` for OTel spans) carrying exact + provider usage and ``metrics.counts.prompt_tokens`` - never a prompt, a turn + or a tool call. Managed ``/api/acp/inference`` relays record no event of their + own (the runtime self-reports instead). +* ``tool.*`` with source ``relay`` are tool executions the relay reconstructs + from model requests; the same call is normally also observed by the ACP proxy. + ACP tool events therefore own a research session's tool metrics whenever the + session has any, and relay tool calls count only in sessions without them. +* ``permission.*`` with source ``relay`` are the built-in agent's own reports + (decisions ``accepted``/``rejected``/``cancelled``/``unavailable``). The ACP + proxy observes the same round-trips, so they count only in sessions without + ACP permission events; ``accepted``/``rejected`` read as ACP's + ``allow``/``reject``. A decision no one made never counts, and neither does + its request: ``decision_scope`` ``policy`` (auto-approval, a suggestion-only + refusal) or ``session_cached`` (an earlier "allow for this session" + answering again), decision ``unavailable`` (the request reached no one), or + a decision a policy without behavioural fields stripped. +* Turn and permission ids are JSON-RPC request ids that restart with every proxy + process, so they are keyed by (research session, emitter, id). +* ``ide.document.changed`` is one IDE document change; its ``payload.count`` is + the number of inserted characters, so edits are counted per event. +* ``ide.file.saved`` with source ``acp`` is an agent ``fs/write_text_file``. + +A value that cannot be computed is ``None`` (JSON ``null``), never ``0``. +""" + +from __future__ import annotations + +import bisect +import math +import re +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from datetime import date, datetime, time, timedelta, timezone +from typing import Any, Iterable, Mapping, Optional, Sequence + +from research.telemetry.chat_lifecycle import ( + CANCEL_METHOD, + CHAT_END_METHODS, + CHAT_START_METHODS, + REVISE_OPTION_ID, + REVISED_DECISION, +) +from research.telemetry.enums import CanonicalEventType, EventSource + +from .models import ( + ArmRow, + AssignmentRow, + DailyEventCount, + DateWindow, + EnrollmentRow, + EventRow, + SessionRow, + StudyFrame, +) + +__all__ = [ + "ANALYTIC_EVENT_TYPES", + "ChatSpan", + "FALLBACK_MAX_CONTEXT_TOKENS", + "METRIC_KEYS", + "ParticipantAnalysis", + "TIMELINE_EXCLUDED_EVENT_TYPES", + "Turn", + "ToolCall", + "analyze_participant", + "build_participant_detail", + "build_chats", + "build_participants", + "build_study_summary", + "build_turns", + "context_block", + "display_tool_name", + "effective_context_cap", + "iso", + "median", + "metric_summary", + "percentile", + "session_seconds", + "transitions", +] + +# -- vocabulary --------------------------------------------------------------- + +PROMPT_EVENT = CanonicalEventType.AGENT_MESSAGE_STARTED.value +COMPLETION_EVENT = CanonicalEventType.AGENT_MESSAGE_COMPLETED.value +CANCEL_EVENT = CanonicalEventType.INTERACTION_COMPLETED.value +CHAT_STARTED_EVENT = CanonicalEventType.INTERACTION_STARTED.value +TOOL_CREATED_EVENT = CanonicalEventType.TOOL_CREATED.value +TOOL_STARTED_EVENT = CanonicalEventType.TOOL_STARTED.value +TOOL_COMPLETED_EVENT = CanonicalEventType.TOOL_COMPLETED.value +TOOL_FAILED_EVENT = CanonicalEventType.TOOL_FAILED.value +TOOL_EVENTS = frozenset( + {TOOL_CREATED_EVENT, TOOL_STARTED_EVENT, TOOL_COMPLETED_EVENT, TOOL_FAILED_EVENT} +) +PERMISSION_REQUESTED_EVENT = CanonicalEventType.PERMISSION_REQUESTED.value +PERMISSION_DECIDED_EVENT = CanonicalEventType.PERMISSION_DECIDED.value +PERMISSION_EVENTS = frozenset({PERMISSION_REQUESTED_EVENT, PERMISSION_DECIDED_EVENT}) +PLAN_EVENT = CanonicalEventType.PLAN_UPDATED.value +USAGE_EVENT = CanonicalEventType.USAGE_UPDATED.value +ERROR_EVENTS = frozenset( + { + CanonicalEventType.AGENT_ERROR.value, + CanonicalEventType.SYSTEM_AGENT_CRASHED.value, + CanonicalEventType.SYSTEM_PROXY_ERROR.value, + } +) +FILE_SAVED_EVENT = CanonicalEventType.IDE_FILE_SAVED.value +DOCUMENT_CHANGED_EVENT = CanonicalEventType.IDE_DOCUMENT_CHANGED.value + +#: Every event type the metric set reads (streamed message chunks excluded). +ANALYTIC_EVENT_TYPES = frozenset( + { + PROMPT_EVENT, + COMPLETION_EVENT, + CANCEL_EVENT, + CHAT_STARTED_EVENT, + PERMISSION_REQUESTED_EVENT, + PERMISSION_DECIDED_EVENT, + PLAN_EVENT, + FILE_SAVED_EVENT, + } + | TOOL_EVENTS + | ERROR_EVENTS +) +#: High-volume event types never listed on the replay timeline (streamed +#: message chunks are excluded as well). They still count in ``ide_edits`` / +#: presence through the per-day aggregates. +TIMELINE_EXCLUDED_EVENT_TYPES = frozenset({DOCUMENT_CHANGED_EVENT, USAGE_EVENT}) + +RELAY_SOURCE = EventSource.RELAY.value +ACP_SOURCE = EventSource.ACP.value +IDE_SOURCE = EventSource.IDE.value + +EDIT_TOOL_KIND = "edit" +OTHER_TOOL_KIND = "other" +FAILED_STATUSES = frozenset({"failed", "error"}) +CANCELLED_STOP_REASON = "cancelled" +UNKNOWN = "unknown" +#: Scopes of self-reported decisions no one was asked for: the approval policy, +#: or an earlier "allow for this session" answering again. +UNASKED_DECISION_SCOPES = frozenset({"policy", "session_cached"}) +#: A self-reported approval request that reached no one. +UNANSWERED_DECISION = "unavailable" +#: The built-in agent's decision vocabulary, read as the ACP one. ``revise`` +#: is the "Revise…" answer (keep some hunks, instructions for the rest). +REVISE_DECISION = "revise" +RELAY_DECISIONS = {"accepted": "allow", "rejected": "reject", REVISED_DECISION: REVISE_DECISION} +#: Chat-number buckets of the history-dependence view. +CHAT_ORDINAL_BUCKETS = ("1", "2", "3", "4+") +#: End reason of a chat whose end was never observed. +OPEN_CHAT = "open" +TERMINAL_SESSION_STATES = frozenset({"ended", "revoked"}) +#: Closes the server stamps when it notices them: an idle timeout or an expired +#: resume grace once the participant comes back, a withdrawal, a stopped study. +#: That can be days after the IDE last reported, so such a session ends at its +#: last heartbeat at the latest. +SERVER_CLOSE_REASONS = frozenset({"idle_timeout", "resume_grace_expired", "revoked", "STUDY_STOPPED"}) +ACTIVE_ENROLLMENT_STATUS = "ACTIVE" +#: A participant is ``ACTIVE`` when their last event is this recent. +ACTIVE_WINDOW = timedelta(days=7) + +#: Mirrors ``FALLBACK_MAX_CONTEXT_TOKENS`` in ``src/backend/routers/acp/__init__.py`` +#: (the managed runtime's context budget when a profile sets none). Kept local: +#: importing the ACP router module would drag in the whole ACP surface. +FALLBACK_MAX_CONTEXT_TOKENS = 32_000 + +TOP_TOOLS_PARTICIPANT = 15 +TOP_TOOLS_STUDY = 20 +MAX_SESSIONS_LIST = 50 +MAX_TURNS = 100 +MAX_CHATS = 100 +MAX_TIMELINE = 200 + +#: A tool identifier (``read_file``, ``developer__shell``, ``mcp__github__search``). +#: No spaces, dots or slashes: a title describes an action on a file or a +#: command ("Read src/app.py", "Run git status"), and a bare file name such as +#: "secrets.env" must not pass as a tool name either. +_TOOL_IDENTIFIER = re.compile(r"[A-Za-z][A-Za-z0-9_:-]{0,63}") + +#: Participant-level metric set M, in display order. +METRIC_KEYS: tuple[str, ...] = ( + "prompts", + "active_days", + "session_hours", + "prompts_per_session_hour", + "tool_calls_per_prompt", + "tool_failure_rate", + "auto_run_share", + "permission_denial_rate", + "median_permission_wait_seconds", + "cancel_rate", + "median_turn_seconds", + "tokens_per_prompt", + "errors_per_prompt", + "agent_writes_per_prompt", + "seconds_to_first_agent_edit", + "ide_edits_per_session_hour", + "plan_completion_rate", + "revision_rate", + "reprompt_after_rejection_rate", + "chats_opened", + "prompts_per_chat", +) +#: Count-type metrics include every assigned participant (a participant without +#: telemetry contributes 0); every other metric excludes participants without +#: telemetry and uses null-free value lists. +COUNT_METRIC_KEYS = frozenset({"prompts", "active_days", "session_hours"}) + +_UTC_MIN = datetime.min.replace(tzinfo=timezone.utc) +_UTC_MAX = datetime.max.replace(tzinfo=timezone.utc) + + +# -- small helpers ------------------------------------------------------------ + + +def as_utc(value: Optional[datetime]) -> Optional[datetime]: + """Return ``value`` as an aware UTC datetime (naive values are UTC).""" + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def iso(value: Optional[datetime]) -> Optional[str]: + """ISO-8601 UTC with a ``Z`` suffix, or ``None``.""" + moment = as_utc(value) + if moment is None: + return None + return moment.isoformat().replace("+00:00", "Z") + + +def _round(value: Optional[float], digits: int) -> Optional[float]: + if value is None: + return None + return round(float(value), digits) + + +def _r3(value: Optional[float]) -> Optional[float]: + return _round(value, 3) + + +def _r1(value: Optional[float]) -> Optional[float]: + return _round(value, 1) + + +def percentile(values: Iterable[float], q: float) -> Optional[float]: + """Linear-interpolation percentile (numpy's default); ``None`` when empty.""" + data = sorted(float(value) for value in values) + if not data: + return None + if len(data) == 1: + return data[0] + position = (len(data) - 1) * q + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return data[lower] + return data[lower] + (data[upper] - data[lower]) * (position - lower) + + +def median(values: Iterable[float]) -> Optional[float]: + return percentile(values, 0.5) + + +def _mean(values: Sequence[float]) -> Optional[float]: + if not values: + return None + return sum(values) / len(values) + + +def _ratio(numerator: Optional[float], denominator: Optional[float]) -> Optional[float]: + if numerator is None or denominator is None or denominator == 0: + return None + return numerator / denominator + + +def order_key(row: EventRow) -> tuple[datetime, str, int]: + """Deterministic event order: time, then the emitter's own sequence.""" + return (row.occurred_at, row.emitter_id or "", row.emitter_sequence or 0) + + +def _utc_day(value: datetime) -> date: + return as_utc(value).date() # type: ignore[union-attr] + + +def is_message_chunk(row: EventRow) -> bool: + """A streamed assistant/thought chunk (never a prompt).""" + return row.event_type == PROMPT_EVENT and ( + row.message_kind is not None or (row.lifecycle_state or "") == "started" + ) + + +def is_prompt(row: EventRow) -> bool: + """A ``session/prompt`` observation: one user prompt and one turn start.""" + return ( + row.event_type == PROMPT_EVENT + and row.source == ACP_SOURCE + and not is_message_chunk(row) + ) + + +def is_turn_completion(row: EventRow) -> bool: + """An ACP prompt response (never a relay-observed model call).""" + return row.event_type == COMPLETION_EVENT and row.source == ACP_SOURCE + + +def is_model_call(row: EventRow) -> bool: + """One relay-observed / self-reported provider model call.""" + return row.event_type == COMPLETION_EVENT and row.source == RELAY_SOURCE + + +def is_cancel(row: EventRow) -> bool: + """An ACP ``session/cancel`` (a user interrupt). + + Chat ends share the event type but carry lifecycle ``completed`` (and an + ``acp_method``/``end_reason``); they are never interrupts. + """ + return ( + row.event_type == CANCEL_EVENT + and row.source == ACP_SOURCE + and row.lifecycle_state in (None, "cancelled") + and row.acp_method in (None, CANCEL_METHOD) + and row.end_reason is None + ) + + +def is_chat_start(row: EventRow) -> bool: + """A chat opened or reattached (``session/new|fork|load|resume``).""" + return ( + row.event_type == CHAT_STARTED_EVENT + and row.source == ACP_SOURCE + and row.acp_method in CHAT_START_METHODS + ) + + +def is_chat_end(row: EventRow) -> bool: + """A chat closed by the client or its agent process ending.""" + return ( + row.event_type == CANCEL_EVENT + and row.source == ACP_SOURCE + and row.lifecycle_state == "completed" + and (row.acp_method in CHAT_END_METHODS or row.end_reason is not None) + ) + + +def relay_decision_made(row: EventRow) -> bool: + """Whether a relay ``permission.decided`` row is a decision someone made. + + The approval policy and a cached "allow for this session" asked no one, + ``unavailable`` reached no one, and a row whose decision a policy stripped + cannot show it was either. + """ + return ( + row.decision is not None + and row.decision != UNANSWERED_DECISION + and row.decision_scope not in UNASKED_DECISION_SCOPES + ) + + +@dataclass(frozen=True) +class PermissionFilter: + """Which permission events count: each round-trip once, from the source + that owns its session, and only when someone was asked.""" + + acp_sessions: frozenset = frozenset() + #: (session, tool call) of relay decisions no one made: their relay + #: requests don't count either. + unasked_relay_calls: frozenset = frozenset() + + @classmethod + def from_rows(cls, rows: Iterable[EventRow]) -> "PermissionFilter": + acp_sessions: set = set() + unasked: set = set() + for row in rows: + if row.event_type not in PERMISSION_EVENTS: + continue + if row.source == ACP_SOURCE: + acp_sessions.add(row.session_id) + elif ( + row.source == RELAY_SOURCE + and row.event_type == PERMISSION_DECIDED_EVENT + and row.tool_call_id + and not relay_decision_made(row) + ): + unasked.add((row.session_id, row.tool_call_id)) + return cls(frozenset(acp_sessions), frozenset(unasked)) + + def counts(self, row: EventRow) -> bool: + if row.event_type not in PERMISSION_EVENTS: + return False + if row.source != RELAY_SOURCE: + return True + if row.session_id in self.acp_sessions: + return False + if row.event_type == PERMISSION_DECIDED_EVENT: + return relay_decision_made(row) + return (row.session_id, row.tool_call_id) not in self.unasked_relay_calls + + +def permission_decision(row: EventRow) -> Optional[str]: + """A ``permission.decided`` outcome in the ACP vocabulary (``None`` if absent). + + The proxy sees "Revise…" as a reject-kind option; its id tells them apart. + """ + if row.selected_option_id == REVISE_OPTION_ID: + return REVISE_DECISION + if row.decision is None: + return None + return RELAY_DECISIONS.get(row.decision, row.decision) + + +def _in_window(moment: datetime, window: Optional[DateWindow]) -> bool: + if window is None or window.is_open: + return True + lower, upper = window.bounds() + moment = as_utc(moment) # type: ignore[assignment] + if lower is not None and moment < lower: + return False + if upper is not None and moment >= upper: + return False + return True + + +def _day_in_window(day: date, window: Optional[DateWindow]) -> bool: + if window is None: + return True + if window.start is not None and day < window.start: + return False + if window.end is not None and day > window.end: + return False + return True + + +# -- sessions ----------------------------------------------------------------- + + +def session_interval( + row: SessionRow, window: Optional[DateWindow] = None +) -> Optional[tuple[datetime, datetime]]: + """``[opened_at, coalesce(closed, last activity, last heartbeat)]`` clipped. + + ``None`` for a session that never opened. A missing or earlier end yields a + zero-length interval (negatives are clamped to 0). A session the server + closed on its own (``SERVER_CLOSE_REASONS``) ends at its last heartbeat at + the latest. + """ + start = as_utc(row.opened_at) + if start is None: + return None + end = as_utc(row.closed_at or row.last_activity_at or row.last_heartbeat_at) + heartbeat = as_utc(row.last_heartbeat_at) + if row.close_reason in SERVER_CLOSE_REASONS and heartbeat is not None and end is not None: + end = min(end, heartbeat) + if end is None or end < start: + end = start + if window is not None and not window.is_open: + lower, upper = window.bounds() + if lower is not None and start < lower: + start = lower + if upper is not None and end > upper: + end = upper + if end < start: + end = start + return start, end + + +def session_seconds(row: SessionRow, window: Optional[DateWindow] = None) -> float: + interval = session_interval(row, window) + if interval is None: + return 0.0 + return max(0.0, (interval[1] - interval[0]).total_seconds()) + + +def split_seconds_by_day(start: datetime, end: datetime) -> dict[date, float]: + """Split ``[start, end)`` into UTC calendar days.""" + seconds: dict[date, float] = {} + cursor = as_utc(start) + stop = as_utc(end) + if cursor is None or stop is None: + return seconds + while cursor < stop: + midnight = datetime.combine( + cursor.date() + timedelta(days=1), time.min, tzinfo=timezone.utc + ) + segment_end = min(stop, midnight) + seconds[cursor.date()] = seconds.get(cursor.date(), 0.0) + ( + segment_end - cursor + ).total_seconds() + cursor = segment_end + return seconds + + +def session_opened_in_window(row: SessionRow, window: Optional[DateWindow]) -> bool: + """Without a window every session counts; with one, by its opening date.""" + if window is None or window.is_open: + return True + opened = as_utc(row.opened_at) + return opened is not None and _in_window(opened, window) + + +# -- turns and tool calls ------------------------------------------------------- + + +@dataclass +class ToolCall: + """One distinct tool call assembled from its lifecycle events.""" + + source: str + session_id: Optional[str] + emitter_id: str + tool_call_id: Optional[str] + first_at: datetime + first_order: tuple + turn_ref: Optional[str] = None + tool_name: Optional[str] = None + tool_kind: Optional[str] = None + last_event_type: str = "" + last_status: Optional[str] = None + completed_at: Optional[datetime] = None + terminal_at: Optional[datetime] = None + latency_ms: Optional[int] = None + requested_permission: bool = False + turn: Optional["Turn"] = None + + @property + def is_relay(self) -> bool: + return self.source == RELAY_SOURCE + + @property + def kind(self) -> str: + return self.tool_kind or OTHER_TOOL_KIND + + @property + def failed(self) -> bool: + """Failed when the final status event is ``tool.failed``/status failed.""" + return self.last_event_type == TOOL_FAILED_EVENT or ( + self.last_status in FAILED_STATUSES + ) + + @property + def is_agent_write(self) -> bool: + return self.kind == EDIT_TOOL_KIND and not self.failed + + @property + def duration_ms(self) -> Optional[float]: + """Observed lifecycle duration, else a reported ``latency_ms``.""" + if self.terminal_at is not None and self.terminal_at > self.first_at: + return (self.terminal_at - self.first_at).total_seconds() * 1000.0 + if self.latency_ms is not None and self.latency_ms >= 0: + return float(self.latency_ms) + return None + + +@dataclass +class Turn: + """One prompt turn: ``agent.message.started`` to its prompt response.""" + + session_id: Optional[str] + emitter_id: str + turn_id: Optional[str] + started_at: datetime + start_order: tuple + #: The ACP chat of the prompt (``None`` on telemetry without chat ids). + chat_id: Optional[str] = None + completed_at: Optional[datetime] = None + stop_reason: Optional[str] = None + completion_usage: Optional[int] = None + relay_usage: Optional[int] = None + usage_tokens: Optional[int] = None + matched_by_turn_id: bool = False + tool_calls: list[ToolCall] = field(default_factory=list) + permission_requests: int = 0 + cancel_events: int = 0 + last_plan: Optional[EventRow] = None + + def complete(self, completion: EventRow, *, by_turn_id: bool) -> None: + self.completed_at = completion.occurred_at + self.stop_reason = completion.stop_reason + self.completion_usage = completion.usage_tokens + self.matched_by_turn_id = by_turn_id + + @property + def completed(self) -> bool: + return self.completed_at is not None + + @property + def duration_seconds(self) -> Optional[float]: + if self.completed_at is None: + return None + return max(0.0, (self.completed_at - self.started_at).total_seconds()) + + @property + def cancelled(self) -> bool: + return self.stop_reason == CANCELLED_STOP_REASON or self.cancel_events > 0 + + @property + def tool_failures(self) -> int: + return sum(1 for call in self.tool_calls if call.failed) + + @property + def plan_completion(self) -> Optional[float]: + plan = self.last_plan + if plan is None or not plan.plan_size or plan.plan_completed is None: + return None + return min(1.0, max(0.0, plan.plan_completed / plan.plan_size)) + + +def build_turns(prompts: Iterable[EventRow], completions: Iterable[EventRow]) -> list[Turn]: + """Pair prompts with their prompt responses. + + Pass 1 matches the same (research session, emitter, turn id). Pass 2 pairs a + still-open turn with the next unmatched completion in the same research + session when either side lacks a turn id, never past the session's next + prompt and never across two different known turn ids. A prompt without a + completion stays an open (incomplete) turn. + """ + ordered_prompts = sorted(prompts, key=order_key) + ordered_completions = sorted(completions, key=order_key) + turns = [ + Turn( + session_id=prompt.session_id, + emitter_id=prompt.emitter_id, + turn_id=prompt.turn_id, + started_at=prompt.occurred_at, + start_order=order_key(prompt), + chat_id=prompt.chat_id, + ) + for prompt in ordered_prompts + ] + + keyed: dict[tuple, list[EventRow]] = defaultdict(list) + for completion in ordered_completions: + if completion.turn_id is not None: + keyed[ + (completion.session_id, completion.emitter_id, completion.turn_id) + ].append(completion) + used: set[int] = set() + for turn in turns: + if turn.turn_id is None: + continue + for completion in keyed.get((turn.session_id, turn.emitter_id, turn.turn_id), ()): + if id(completion) in used or order_key(completion) < turn.start_order: + continue + turn.complete(completion, by_turn_id=True) + used.add(id(completion)) + break + + turns_by_session: dict[Optional[str], list[Turn]] = defaultdict(list) + for turn in turns: + turns_by_session[turn.session_id].append(turn) + completions_by_session: dict[Optional[str], list[EventRow]] = defaultdict(list) + for completion in ordered_completions: + if id(completion) not in used: + completions_by_session[completion.session_id].append(completion) + for session_id, session_turns in turns_by_session.items(): + pending = completions_by_session.get(session_id, []) + if not pending: + continue + for position, turn in enumerate(session_turns): + if turn.completed: + continue + next_start = ( + session_turns[position + 1].start_order + if position + 1 < len(session_turns) + else None + ) + for completion in pending: + if id(completion) in used: + continue + completion_order = order_key(completion) + if completion_order < turn.start_order: + continue + if next_start is not None and completion_order >= next_start: + break + if turn.turn_id is not None and completion.turn_id is not None: + continue + turn.complete(completion, by_turn_id=False) + used.add(id(completion)) + break + return turns + + +class TurnIndex: + """Attributes events to turns. + + An ACP event carrying a turn id belongs to the turn with the same (session, + emitter, turn id); one without a turn id belongs to the turn of the same + research session whose window contains it. A relay event (different clock, + no turn id) belongs to the latest turn of its research session started at or + before it. + """ + + def __init__(self, turns: Iterable[Turn]) -> None: + self.by_key: dict[tuple, Turn] = {} + self._starts: dict[Optional[str], list[tuple]] = defaultdict(list) + self._turns: dict[Optional[str], list[Turn]] = defaultdict(list) + for turn in sorted(turns, key=lambda item: item.start_order): + if turn.turn_id is not None: + self.by_key.setdefault((turn.session_id, turn.emitter_id, turn.turn_id), turn) + self._starts[turn.session_id].append(turn.start_order) + self._turns[turn.session_id].append(turn) + + def latest_in_session( + self, session_id: Optional[str], moment: datetime + ) -> Optional[Turn]: + starts = self._starts.get(session_id) + if not starts: + return None + position = bisect.bisect_right(starts, (moment, "￿", math.inf)) + return self._turns[session_id][position - 1] if position else None + + def within_session( + self, session_id: Optional[str], moment: datetime + ) -> Optional[Turn]: + turn = self.latest_in_session(session_id, moment) + if turn is None: + return None + if turn.completed_at is not None and moment > turn.completed_at: + return None + return turn + + def for_event(self, row: EventRow) -> Optional[Turn]: + if row.source == RELAY_SOURCE: + return self.latest_in_session(row.session_id, row.occurred_at) + if row.turn_id is not None: + return self.by_key.get((row.session_id, row.emitter_id, row.turn_id)) + return self.within_session(row.session_id, row.occurred_at) + + def for_tool_call(self, call: ToolCall) -> Optional[Turn]: + if call.is_relay: + return self.latest_in_session(call.session_id, call.first_at) + if call.turn_ref is not None: + return self.by_key.get((call.session_id, call.emitter_id, call.turn_ref)) + return self.within_session(call.session_id, call.first_at) + + +def _tool_call_key(row: EventRow) -> Optional[tuple]: + if row.source == RELAY_SOURCE: + # Each relay event is one execution; an id merges the relay's emitters. + if row.tool_call_id: + return (True, row.session_id, "", row.tool_call_id) + return (True, row.session_id, row.emitter_id, f"#{row.emitter_sequence}") + if not row.tool_call_id: + # e.g. ``terminal/create`` has no tool call id: it belongs to (and would + # double count) the execute tool call that owns the terminal. + return None + return (False, row.session_id, row.emitter_id, row.tool_call_id) + + +def build_tool_calls(rows: Iterable[EventRow]) -> dict[tuple, ToolCall]: + """Group ``tool.*`` events into distinct tool calls (rows in event order).""" + calls: dict[tuple, ToolCall] = {} + for row in sorted(rows, key=order_key): + if row.event_type not in TOOL_EVENTS: + continue + key = _tool_call_key(row) + if key is None: + continue + call = calls.get(key) + if call is None: + call = ToolCall( + source=row.source, + session_id=row.session_id, + emitter_id=row.emitter_id, + tool_call_id=row.tool_call_id, + first_at=row.occurred_at, + first_order=order_key(row), + ) + calls[key] = call + if call.turn_ref is None and row.turn_id is not None: + call.turn_ref = row.turn_id + if call.tool_name is None and row.tool_name: + call.tool_name = row.tool_name + if call.tool_kind is None and row.tool_kind: + call.tool_kind = str(row.tool_kind).strip().lower() or None + call.last_event_type = row.event_type + call.last_status = (row.status or "").strip().lower() or None + if row.event_type == TOOL_COMPLETED_EVENT and call.completed_at is None: + call.completed_at = row.occurred_at + if row.event_type in (TOOL_COMPLETED_EVENT, TOOL_FAILED_EVENT): + call.terminal_at = row.occurred_at + if row.latency_ms is not None: + call.latency_ms = row.latency_ms + return calls + + +def count_agent_writes( + tool_calls: Iterable[ToolCall], agent_saves: Iterable[EventRow], index: TurnIndex +) -> int: + """Files the agent wrote: per turn, the larger of its two observations. + + One write can be seen twice: as a successful ``edit`` tool call and as the + ``fs/write_text_file`` request that call made through the IDE + (``ide.file.saved`` from the ACP proxy). Agents that write to disk + themselves produce only the call; a write outside any tool call produces + only the save. Per turn (else per research session) the count is the + larger of the two, never their sum. + """ + edits: Counter = Counter() + saves: Counter = Counter() + for call in tool_calls: + if call.is_agent_write: + turn = index.for_tool_call(call) + edits[id(turn) if turn is not None else ("session", call.session_id)] += 1 + for save in agent_saves: + turn = index.for_event(save) + saves[id(turn) if turn is not None else ("session", save.session_id)] += 1 + return sum(max(edits[key], saves[key]) for key in set(edits) | set(saves)) + + +def transitions(turns: Iterable[Turn]) -> list[dict[str, Any]]: + """Consecutive ``tool_kind`` pairs within turns, with lift. + + Each turn's tool calls are ordered by first occurrence. ``lift`` is + ``P(to | from) / P(to)`` over all counted transitions (``None`` when a + denominator is 0). + """ + pairs: Counter[tuple[str, str]] = Counter() + for turn in turns: + kinds = [call.kind for call in sorted(turn.tool_calls, key=lambda call: call.first_order)] + for source_kind, target_kind in zip(kinds, kinds[1:]): + pairs[(source_kind, target_kind)] += 1 + total = sum(pairs.values()) + from_totals: Counter[str] = Counter() + to_totals: Counter[str] = Counter() + for (source_kind, target_kind), count in pairs.items(): + from_totals[source_kind] += count + to_totals[target_kind] += count + rows: list[dict[str, Any]] = [] + for (source_kind, target_kind), count in pairs.items(): + conditional = _ratio(count, from_totals[source_kind]) + marginal = _ratio(to_totals[target_kind], total) + lift = _ratio(conditional, marginal) if conditional is not None else None + rows.append( + {"from": source_kind, "to": target_kind, "count": count, "lift": _r3(lift)} + ) + rows.sort(key=lambda row: (-row["count"], row["from"], row["to"])) + return rows + + +# -- participant analysis --------------------------------------------------------- + + +# -- chats --------------------------------------------------------------------- + + +@dataclass +class ChatSpan: + """One ACP chat of a participant, from its lifecycle events and turns.""" + + chat_id: str + first_at: datetime + last_at: datetime + research_session_ids: set = field(default_factory=set) + #: ``new``/``fork``/``load``/``resume`` from the first start seen, else None. + start_kind: Optional[str] = None + started_at: Optional[datetime] = None + ended_at: Optional[datetime] = None + #: ``close`` or a proxy end reason (``host_closed``, ``agent_exited``, …). + end_reason: Optional[str] = None + reopen_count: int = 0 + prompts: int = 0 + tool_calls: int = 0 + cancels: int = 0 + rejections: int = 0 + revisions: int = 0 + usage_tokens: Optional[int] = None + ordinal: int = 0 + previous_end_reason: Optional[str] = None + gap_since_previous_seconds: Optional[float] = None + + @property + def opened_at(self) -> datetime: + return self.started_at or self.first_at + + @property + def used(self) -> bool: + """Whether the participant sent a prompt in the chat. IntelliJ also + starts an agent for a chat it only creates or shows (a new empty chat, + the chat it switches to after a delete), so such chats are listed but + neither numbered nor counted.""" + return self.prompts > 0 + + @property + def end_label(self) -> str: + """How the chat ended: a reason, ``open`` when it is still going, or + ``unknown`` on telemetry without chat lifecycle events.""" + if self.end_reason: + return self.end_reason + return OPEN_CHAT if self.start_kind else UNKNOWN + + +def build_chats( + rows: Sequence[EventRow], turns: Sequence[Turn], decisions: Sequence[EventRow] +) -> list[ChatSpan]: + """A participant's chats in opening order, each linked to the one before. + + Chats are keyed by the ACP chat id the proxy stamps on its events. Turns + count in the chat of their prompt, permission decisions in their own chat. + A reopened chat (``session/load|resume`` after an end) is open again. Only + used chats (with a prompt) are numbered and linked to the one before. + """ + spans: dict[str, ChatSpan] = {} + + def span(chat_id: str, at: datetime) -> ChatSpan: + current = spans.get(chat_id) + if current is None: + current = spans[chat_id] = ChatSpan(chat_id=chat_id, first_at=at, last_at=at) + else: + current.first_at = min(current.first_at, at) + current.last_at = max(current.last_at, at) + return current + + for row in rows: + if not row.chat_id or row.source != ACP_SOURCE: + continue + current = span(row.chat_id, row.occurred_at) + if row.session_id: + current.research_session_ids.add(row.session_id) + if is_chat_start(row): + kind = CHAT_START_METHODS[row.acp_method or ""] + if kind in ("load", "resume"): + current.reopen_count += 1 + if current.start_kind is None: + current.start_kind = kind + current.started_at = row.occurred_at + if current.ended_at is not None and row.occurred_at >= current.ended_at: + current.ended_at = None + current.end_reason = None + elif is_chat_end(row): + reason = "close" if row.acp_method in CHAT_END_METHODS else row.end_reason + # The client's own close outranks the process ending after it. + if current.end_reason is None or reason == "close": + current.end_reason = reason + current.ended_at = row.occurred_at + for turn in turns: + if not turn.chat_id: + continue + current = span(turn.chat_id, turn.started_at) + current.prompts += 1 + current.tool_calls += len(turn.tool_calls) + current.cancels += 1 if turn.cancelled else 0 + if turn.usage_tokens is not None: + current.usage_tokens = (current.usage_tokens or 0) + turn.usage_tokens + for decision in decisions: + if not decision.chat_id: + continue + outcome = permission_decision(decision) + current = span(decision.chat_id, decision.occurred_at) + if outcome == "reject": + current.rejections += 1 + elif outcome == REVISE_DECISION: + current.revisions += 1 + + ordered = sorted(spans.values(), key=lambda chat: (chat.opened_at, chat.chat_id)) + previous: Optional[ChatSpan] = None + for ordinal, chat in enumerate((chat for chat in ordered if chat.used), start=1): + chat.ordinal = ordinal + if previous is not None: + chat.previous_end_reason = previous.end_label + gap = (chat.opened_at - (previous.ended_at or previous.last_at)).total_seconds() + chat.gap_since_previous_seconds = max(0.0, gap) + previous = chat + return ordered + + +def reprompted_rejections( + decisions: Iterable[EventRow], prompts: Iterable[EventRow] +) -> tuple[int, int]: + """``(followed, total)``: rejections in a chat, and those followed by + another prompt in the same chat (the participant re-prompted instead of + leaving it there).""" + prompt_times: dict[str, list[datetime]] = defaultdict(list) + for prompt in prompts: + if prompt.chat_id: + prompt_times[prompt.chat_id].append(prompt.occurred_at) + for moments in prompt_times.values(): + moments.sort() + followed = total = 0 + for decision in decisions: + if not decision.chat_id or permission_decision(decision) != "reject": + continue + total += 1 + moments = prompt_times.get(decision.chat_id, []) + if bisect.bisect_right(moments, decision.occurred_at) < len(moments): + followed += 1 + return followed, total + + +@dataclass +class ParticipantAnalysis: + """Every intermediate needed for one participant's rows and metrics.""" + + prompts: list[EventRow] + turns: list[Turn] + completions: list[EventRow] + model_calls: list[EventRow] + tool_calls: list[ToolCall] + call_lookup: dict[tuple, ToolCall] + permission_requests: list[EventRow] + decision_counts: Counter + permission_waits: list[float] + cancels: list[EventRow] + errors: list[EventRow] + agent_saves: list[EventRow] + agent_file_writes: int + usage_tokens: Optional[int] + session_seconds: float + session_seconds_by_session: dict[str, float] + session_seconds_by_day: dict[date, float] + ide_edits: int + ide_edits_by_day: dict[date, int] + events_by_day: dict[date, int] + event_count: int + first_event_at: Optional[datetime] + last_event_at: Optional[datetime] + first_edit_seconds: list[float] + permission_filter: PermissionFilter = field(default_factory=PermissionFilter) + chats: list[ChatSpan] = field(default_factory=list) + #: Whether the participant's agent ever offered "Revise…" (built-in arms). + revise_offered: bool = False + #: Rejections followed by another prompt in the same chat / all rejections in a chat. + reprompted_rejections: int = 0 + chat_rejections: int = 0 + + @property + def has_telemetry(self) -> bool: + return self.event_count > 0 + + @property + def acp_tool_calls(self) -> list[ToolCall]: + return [call for call in self.tool_calls if not call.is_relay] + + @property + def tool_failures(self) -> int: + return sum(1 for call in self.tool_calls if call.failed) + + @property + def active_days(self) -> int: + return len({_utc_day(prompt.occurred_at) for prompt in self.prompts}) + + @property + def stop_reason_counts(self) -> Counter: + return Counter( + (completion.stop_reason or UNKNOWN) for completion in self.completions + ) + + @property + def completed_turn_seconds(self) -> list[float]: + return [ + turn.duration_seconds + for turn in self.turns + if turn.duration_seconds is not None + ] + + def metrics(self) -> dict[str, Optional[float]]: + """The participant-level metric set M (floats rounded to 3 decimals).""" + prompts = len(self.prompts) + session_hours = self.session_seconds / 3600.0 + tool_calls = len(self.tool_calls) + acp_calls = self.acp_tool_calls + auto_run = sum(1 for call in acp_calls if not call.requested_permission) + allow = self.decision_counts.get("allow", 0) + reject = self.decision_counts.get("reject", 0) + revise = self.decision_counts.get(REVISE_DECISION, 0) + prompted_chats = [chat for chat in self.chats if chat.prompts] + turns = len(self.turns) + cancelled = sum(1 for turn in self.turns if turn.cancelled) + usage = [ + turn.usage_tokens + for turn in self.turns + if turn.completed and turn.usage_tokens is not None + ] + plan_rates = [ + rate + for rate in (turn.plan_completion for turn in self.turns) + if rate is not None + ] + return { + "prompts": prompts, + "active_days": self.active_days, + "session_hours": _r3(session_hours), + "prompts_per_session_hour": _r3(_ratio(prompts, session_hours)), + "tool_calls_per_prompt": _r3(_ratio(tool_calls, prompts)), + "tool_failure_rate": _r3(_ratio(self.tool_failures, tool_calls)), + "auto_run_share": _r3(_ratio(auto_run, len(acp_calls))), + "permission_denial_rate": _r3(_ratio(reject, allow + reject)), + "median_permission_wait_seconds": _r3(median(self.permission_waits)), + "cancel_rate": _r3(_ratio(cancelled, turns)), + "median_turn_seconds": _r3(median(self.completed_turn_seconds)), + "tokens_per_prompt": _r3(_ratio(sum(usage), len(usage)) if usage else None), + "errors_per_prompt": _r3(_ratio(len(self.errors), prompts)), + "agent_writes_per_prompt": _r3(_ratio(self.agent_file_writes, prompts)), + "seconds_to_first_agent_edit": _r3(median(self.first_edit_seconds)), + "ide_edits_per_session_hour": _r3(_ratio(self.ide_edits, session_hours)), + "plan_completion_rate": _r3(_mean(plan_rates)), + # Null, not 0, where the agent never offered "Revise…" (other arms). + "revision_rate": ( + _r3(_ratio(revise, allow + reject + revise)) if self.revise_offered else None + ), + "reprompt_after_rejection_rate": _r3( + _ratio(self.reprompted_rejections, self.chat_rejections) + ), + "chats_opened": self.chats_opened, + "prompts_per_chat": _r3( + _ratio(sum(chat.prompts for chat in prompted_chats), len(prompted_chats)) + ), + } + + @property + def chats_opened(self) -> Optional[int]: + """Chats opened afresh and used (a reopened old chat is not a new one, a + chat without a prompt is not counted); ``None`` when prompts carry no + chat ids, so the count is not observable.""" + if not self.chats: + return None if self.prompts else 0 + return sum( + 1 for chat in self.chats if chat.used and chat.start_kind in (None, "new", "fork") + ) + + +def analyze_participant( + events: Iterable[EventRow], + sessions: Iterable[SessionRow], + daily: Optional[Iterable[DailyEventCount]] = None, + *, + window: Optional[DateWindow] = None, +) -> ParticipantAnalysis: + """Analyse one participant's metadata-only events and sessions. + + ``daily`` (per-date aggregates over *all* retained events) supplies event + presence, first/last event time and IDE edit counts; when it is ``None`` + they are derived from ``events`` (``ide.document.changed`` rows count one + edit each). The builders pass ``None`` for an enrollment without aggregate + rows, which in the database means it has no retained events at all. + """ + rows = sorted( + (row for row in events if _in_window(row.occurred_at, window)), key=order_key + ) + + prompts = [row for row in rows if is_prompt(row)] + completions = [row for row in rows if is_turn_completion(row)] + model_calls = [row for row in rows if is_model_call(row)] + turns = build_turns(prompts, completions) + index = TurnIndex(turns) + + # Tool calls, with ACP/proxy ownership of a session's tool metrics. + call_lookup = build_tool_calls(rows) + # An ACP title is not a tool name. Where the relay reported the same call + # (the built-in agent does), its tool identifier names the ACP call. + for call in call_lookup.values(): + if call.is_relay or not call.tool_call_id or display_tool_name(call.tool_name): + continue + relay = call_lookup.get((True, call.session_id, "", call.tool_call_id)) + if relay is not None and display_tool_name(relay.tool_name): + call.tool_name = relay.tool_name + observed_sessions = { + call.session_id for call in call_lookup.values() if not call.is_relay + } + tool_calls = sorted( + ( + call + for call in call_lookup.values() + if not call.is_relay or call.session_id not in observed_sessions + ), + key=lambda call: call.first_order, + ) + for call in tool_calls: + call.turn = index.for_tool_call(call) + if call.turn is not None: + call.turn.tool_calls.append(call) + + # Permissions: each round-trip once, from the source that owns the session. + permission_filter = PermissionFilter.from_rows(rows) + permission_rows = [row for row in rows if permission_filter.counts(row)] + permission_requests = [ + row for row in permission_rows if row.event_type == PERMISSION_REQUESTED_EVENT + ] + decisions = [ + row for row in permission_rows if row.event_type == PERMISSION_DECIDED_EVENT + ] + first_request: dict[tuple, EventRow] = {} + permission_tool_calls: set[tuple] = set() + for request in permission_requests: + if request.permission_id is not None: + first_request.setdefault( + (request.session_id, request.emitter_id, request.permission_id), request + ) + if request.tool_call_id: + permission_tool_calls.add( + (request.session_id, request.emitter_id, request.tool_call_id) + ) + turn = index.for_event(request) + if turn is not None: + turn.permission_requests += 1 + for call in tool_calls: + if not call.is_relay: + call.requested_permission = ( + call.session_id, + call.emitter_id, + call.tool_call_id, + ) in permission_tool_calls + decision_counts: Counter = Counter() + permission_waits: list[float] = [] + decided: set[tuple] = set() + for decision in decisions: + decision_counts[permission_decision(decision) or UNKNOWN] += 1 + if decision.permission_id is None: + continue + key = (decision.session_id, decision.emitter_id, decision.permission_id) + request = first_request.get(key) + if request is None or key in decided: + continue + if decision.occurred_at >= request.occurred_at: + decided.add(key) + permission_waits.append( + (decision.occurred_at - request.occurred_at).total_seconds() + ) + + # User interrupts (``session/cancel``) and plan updates inside turns. + cancels = [row for row in rows if is_cancel(row)] + for cancel in cancels: + turn = index.for_event(cancel) + if turn is not None: + turn.cancel_events += 1 + for plan in (row for row in rows if row.event_type == PLAN_EVENT): + turn = index.for_event(plan) + if turn is not None: + turn.last_plan = plan # rows are in event order: the last one wins + + errors = [row for row in rows if row.event_type in ERROR_EVENTS] + agent_saves = [ + row + for row in rows + if row.event_type == FILE_SAVED_EVENT and row.source == ACP_SOURCE + ] + agent_file_writes = count_agent_writes(tool_calls, agent_saves, index) + + # Token usage: ACP prompt-response usage owns a research session when any is + # reported there; otherwise exact relay model-call usage is used (and + # attributed to the latest turn of the session started before the call). + acp_usage_sessions = { + completion.session_id + for completion in completions + if completion.usage_tokens is not None + } + usage_total: Optional[int] = None + for completion in completions: + if completion.usage_tokens is not None: + usage_total = (usage_total or 0) + completion.usage_tokens + for call in model_calls: + if call.usage_tokens is None or call.session_id in acp_usage_sessions: + continue + usage_total = (usage_total or 0) + call.usage_tokens + turn = index.latest_in_session(call.session_id, call.occurred_at) + if turn is not None: + turn.relay_usage = (turn.relay_usage or 0) + call.usage_tokens + for turn in turns: + turn.usage_tokens = ( + turn.completion_usage + if turn.completion_usage is not None + else turn.relay_usage + ) + + # Chats (ACP chat ids) and what follows a rejection inside one. + chats = build_chats(rows, turns, decisions) + reprompted, chat_rejections = reprompted_rejections(decisions, prompts) + + # Sessions (clipped to the window). Two project windows open at the same + # time are one stretch of the participant's time, so overlapping sessions + # are merged before their time is added up. + session_rows = list(sessions) + seconds_by_session: dict[str, float] = {} + seconds_by_day: dict[date, float] = defaultdict(float) + intervals = [] + for session in session_rows: + seconds_by_session[session.session_id] = session_seconds(session, window) + interval = session_interval(session, window) + if interval is not None: + intervals.append(interval) + merged: list[list[datetime]] = [] + for start, end in sorted(intervals): + if merged and start <= merged[-1][1]: + merged[-1][1] = max(merged[-1][1], end) + else: + merged.append([start, end]) + for start, end in merged: + for day, seconds in split_seconds_by_day(start, end).items(): + seconds_by_day[day] += seconds + + # Presence, first/last event and IDE edits. + events_by_day: dict[date, int] = defaultdict(int) + ide_by_day: dict[date, int] = defaultdict(int) + first_event_at: Optional[datetime] = None + last_event_at: Optional[datetime] = None + if daily is not None: + for bucket in daily: + if not _day_in_window(bucket.day, window): + continue + events_by_day[bucket.day] += bucket.events + if bucket.ide_edits: + ide_by_day[bucket.day] += bucket.ide_edits + if bucket.first_at is not None and ( + first_event_at is None or bucket.first_at < first_event_at + ): + first_event_at = bucket.first_at + if bucket.last_at is not None and ( + last_event_at is None or bucket.last_at > last_event_at + ): + last_event_at = bucket.last_at + else: + for row in rows: + events_by_day[_utc_day(row.occurred_at)] += 1 + if row.event_type == DOCUMENT_CHANGED_EVENT: + ide_by_day[_utc_day(row.occurred_at)] += 1 + if rows: + first_event_at = rows[0].occurred_at + last_event_at = max(row.occurred_at for row in rows) + + # Implementation onset: first agent edit after the session's first prompt. + first_prompt: dict[str, datetime] = {} + for prompt in prompts: + if prompt.session_id is not None: + first_prompt.setdefault(prompt.session_id, prompt.occurred_at) + edits_by_session: dict[Optional[str], list[datetime]] = defaultdict(list) + for call in tool_calls: + if call.is_agent_write and call.completed_at is not None: + edits_by_session[call.session_id].append(call.completed_at) + for save in agent_saves: + edits_by_session[save.session_id].append(save.occurred_at) + first_edit_seconds: list[float] = [] + for session_id, started in first_prompt.items(): + later = [moment for moment in edits_by_session.get(session_id, ()) if moment >= started] + if later: + first_edit_seconds.append((min(later) - started).total_seconds()) + + return ParticipantAnalysis( + prompts=prompts, + turns=turns, + completions=completions, + model_calls=model_calls, + tool_calls=tool_calls, + call_lookup=call_lookup, + permission_requests=permission_requests, + decision_counts=decision_counts, + permission_waits=permission_waits, + cancels=cancels, + errors=errors, + agent_saves=agent_saves, + agent_file_writes=agent_file_writes, + usage_tokens=usage_total, + session_seconds=sum((end - start).total_seconds() for start, end in merged), + session_seconds_by_session=seconds_by_session, + session_seconds_by_day=dict(seconds_by_day), + ide_edits=sum(ide_by_day.values()), + ide_edits_by_day=dict(ide_by_day), + events_by_day=dict(events_by_day), + event_count=sum(events_by_day.values()), + first_event_at=first_event_at, + last_event_at=last_event_at, + first_edit_seconds=first_edit_seconds, + permission_filter=permission_filter, + chats=chats, + revise_offered=( + any(request.offers_revise for request in permission_requests) + or decision_counts.get(REVISE_DECISION, 0) > 0 + ), + reprompted_rejections=reprompted, + chat_rejections=chat_rejections, + ) + + +# -- shared JSON blocks ----------------------------------------------------------- + + +def metric_summary(values: Iterable[Optional[float]]) -> dict[str, Any]: + """``n``/mean/median/p25/p75 plus the sorted null-free values (strip plot).""" + data = sorted(float(value) for value in values if value is not None) + return { + "n": len(data), + "mean": _r3(_mean(data)), + "median": _r3(percentile(data, 0.5)), + "p25": _r3(percentile(data, 0.25)), + "p75": _r3(percentile(data, 0.75)), + "values": [_r3(value) for value in data], + } + + +def _counter_rows(counter: Mapping[str, int], label: str) -> list[dict[str, Any]]: + return [ + {label: key, "count": count} + for key, count in sorted(counter.items(), key=lambda item: (-item[1], item[0])) + if count + ] + + +def chat_rows(chats: Sequence[ChatSpan], limit: int = MAX_CHATS) -> list[dict[str, Any]]: + """The participant's latest chats in opening order (no content); chats + without a prompt have no number (``ordinal`` None, ``used`` False).""" + return [ + { + "chat_id": chat.chat_id, + "ordinal": chat.ordinal or None, + "used": chat.used, + "research_session_ids": sorted(chat.research_session_ids), + "started_at": iso(chat.opened_at), + "ended_at": iso(chat.ended_at), + "last_event_at": iso(chat.last_at), + "start_kind": chat.start_kind or UNKNOWN, + "end_reason": chat.end_label, + "reopen_count": chat.reopen_count, + "prompts": chat.prompts, + "tool_calls": chat.tool_calls, + "cancels": chat.cancels, + "rejections": chat.rejections, + "revisions": chat.revisions, + "usage_tokens": chat.usage_tokens, + "previous_end_reason": chat.previous_end_reason, + "gap_since_previous_seconds": _r1(chat.gap_since_previous_seconds), + } + for chat in list(chats)[-limit:] + ] + + +def chat_summary(analysis: "ParticipantAnalysis") -> dict[str, Any]: + """How a participant's used chats started and ended, how many chats the IDE + opened without a prompt, and how observable that is.""" + chats = [chat for chat in analysis.chats if chat.used] + return { + "chats": len(chats), + "unused_chats": len(analysis.chats) - len(chats), + "started": dict(Counter(chat.start_kind or UNKNOWN for chat in chats)), + "ended": dict(Counter(chat.end_label for chat in chats)), + "unattributed_prompts": sum(1 for turn in analysis.turns if not turn.chat_id), + "lifecycle_coverage": _coverage_state( + sum(1 for chat in chats if chat.start_kind), len(chats) + ), + } + + +def _ordinal_bucket(ordinal: int) -> str: + return CHAT_ORDINAL_BUCKETS[min(max(ordinal, 1), len(CHAT_ORDINAL_BUCKETS)) - 1] + + +def _new_chat_group() -> dict[str, Any]: + return { + "participants": set(), + "chats": 0, + "prompts": 0, + "cancels": 0, + "rejections": 0, + "revisions": 0, + "ends": Counter(), + } + + +def _chat_group_row(group: Mapping[str, Any]) -> dict[str, Any]: + return { + "participants": len(group["participants"]), + "chats": group["chats"], + "prompts_per_chat": _r3(_ratio(group["prompts"], group["chats"])), + "cancel_rate": _r3(_ratio(group["cancels"], group["prompts"])), + "rejections_per_prompt": _r3(_ratio(group["rejections"], group["prompts"])), + "revisions_per_prompt": _r3(_ratio(group["revisions"], group["prompts"])), + "end_reasons": _counter_rows(group["ends"], "end_reason"), + } + + +def chat_history( + analyses: Mapping[str, "ParticipantAnalysis"], + arm_of: Mapping[str, str], + arm_ids: Sequence[str], +) -> dict[str, Any]: + """History dependence, descriptively: used chats grouped by their number in + the participant's history and by how the previous one ended (pooled over + chats, per arm; numbers count from the start of the selected range).""" + by_ordinal: dict[tuple, dict[str, Any]] = defaultdict(_new_chat_group) + by_previous: dict[tuple, dict[str, Any]] = defaultdict(_new_chat_group) + for enrollment_id, analysis in analyses.items(): + arm_id = arm_of.get(enrollment_id) + if arm_id is None: + continue + for chat in analysis.chats: + if not chat.used: + continue + groups = [by_ordinal[(arm_id, _ordinal_bucket(chat.ordinal))]] + if chat.previous_end_reason is not None: + groups.append(by_previous[(arm_id, chat.previous_end_reason)]) + for group in groups: + group["participants"].add(enrollment_id) + group["chats"] += 1 + group["prompts"] += chat.prompts + group["cancels"] += chat.cancels + group["rejections"] += chat.rejections + group["revisions"] += chat.revisions + group["ends"][chat.end_label] += 1 + return { + "by_ordinal": [ + {"profile_id": arm_id, "bucket": bucket, **_chat_group_row(by_ordinal[(arm_id, bucket)])} + for arm_id in arm_ids + for bucket in CHAT_ORDINAL_BUCKETS + if (arm_id, bucket) in by_ordinal + ], + "by_previous_end": [ + {"profile_id": arm_id, "previous_end_reason": reason, **_chat_group_row(by_previous[(arm_id, reason)])} + for arm_id in arm_ids + for reason in sorted({key[1] for key in by_previous if key[0] == arm_id}) + ], + } + + +def tool_kind_rows(calls: Iterable[ToolCall]) -> list[dict[str, Any]]: + stats: dict[str, list[int]] = defaultdict(lambda: [0, 0]) + for call in calls: + stats[call.kind][0] += 1 + if call.failed: + stats[call.kind][1] += 1 + return [ + {"tool_kind": kind, "calls": values[0], "failures": values[1]} + for kind, values in sorted(stats.items(), key=lambda item: (-item[1][0], item[0])) + ] + + +def display_tool_name(name: Any) -> Optional[str]: + """``name`` when it is a tool identifier, else ``None``. + + ACP reports a tool call's human-readable *title* as ``tool_name``, and a + title can name files, quote a search pattern or a whole shell command + (``Read src/app.py``, ``Search "token"``, ``Run git status``). Dashboards + never show a title: the call is named by the relay's identifier for the + same call when there is one (see ``analyze_participant``), and otherwise + counted without a name, under its kind. + """ + if not isinstance(name, str): + return None + candidate = name.strip() + return candidate if _TOOL_IDENTIFIER.fullmatch(candidate) else None + + +def tool_rows( + calls: Iterable[tuple[ToolCall, Optional[str]]], + *, + limit: int, + arm_ids: Optional[Sequence[str]] = None, +) -> list[dict[str, Any]]: + """Top tools by calls; with ``arm_ids`` each row carries per-arm calls. + + Calls whose name cannot be shown (see ``display_tool_name``) are grouped + by kind under a ``None`` name. + """ + grouped: dict[tuple[Optional[str], Optional[str]], list[tuple[ToolCall, Optional[str]]]] = defaultdict(list) + for call, arm_id in calls: + name = display_tool_name(call.tool_name) + grouped[(name, None if name is not None else call.kind)].append((call, arm_id)) + rows: list[dict[str, Any]] = [] + for (name, _kind), members in grouped.items(): + kinds = Counter(call.kind for call, _ in members) + kind = sorted(kinds.items(), key=lambda item: (-item[1], item[0]))[0][0] + durations = [ + call.duration_ms for call, _ in members if call.duration_ms is not None + ] + row: dict[str, Any] = { + "tool_name": name, + "tool_kind": kind, + "calls": len(members), + "failures": sum(1 for call, _ in members if call.failed), + "median_duration_ms": _r1(median(durations)), + } + if arm_ids is not None: + by_arm = {arm_id: 0 for arm_id in arm_ids} + for _, arm_id in members: + if arm_id is not None and arm_id in by_arm: + by_arm[arm_id] += 1 + row["by_arm"] = by_arm + rows.append(row) + rows.sort( + key=lambda row: (-row["calls"], row["tool_name"] is None, row["tool_name"] or "", row["tool_kind"] or "") + ) + return rows[:limit] + + +def effective_context_cap(max_context_tokens: Optional[int]) -> int: + """The frozen ``max_context_tokens``, else the managed-runtime fallback.""" + if ( + isinstance(max_context_tokens, int) + and not isinstance(max_context_tokens, bool) + and max_context_tokens >= 1 + ): + return max_context_tokens + return FALLBACK_MAX_CONTEXT_TOKENS + + +def context_block(model_calls: Sequence[EventRow], cap_tokens: int) -> dict[str, Any]: + """Context-cap compliance over relay model calls' provider prompt tokens. + + A call is over the cap when its ``metrics.counts.prompt_tokens`` exceeds + ``cap_tokens``. Arms whose runtime never uses the relay (external BYOA + runtimes) have no model calls, so their coverage is ``UNAVAILABLE``. + """ + prompt_tokens = [ + call.prompt_tokens for call in model_calls if call.prompt_tokens is not None + ] + over_cap = sum(1 for value in prompt_tokens if value > cap_tokens) + return { + "cap_tokens": cap_tokens, + "model_calls": len(model_calls), + "calls_with_prompt_tokens": len(prompt_tokens), + "prompt_tokens_p50": _r3(percentile(prompt_tokens, 0.5)), + "prompt_tokens_p95": _r3(percentile(prompt_tokens, 0.95)), + "prompt_tokens_max": max(prompt_tokens) if prompt_tokens else None, + "over_cap_calls": over_cap, + "over_cap_share": _r3(_ratio(over_cap, len(prompt_tokens))), + "coverage": _coverage_state(len(prompt_tokens), len(model_calls)), + } + + +def _arm_labels(arm: ArmRow) -> dict[str, Any]: + return { + "profile_id": arm.profile_id, + "name": arm.name, + "model": arm.model, + "framework_version": arm.framework_version, + } + + +def arm_catalogue(frame: StudyFrame) -> list[ArmRow]: + """The study's frozen arms by selection order, plus any assigned extra.""" + arms = sorted( + frame.arms, + key=lambda arm: ( + arm.selection_order is None, + arm.selection_order or 0, + arm.profile_id, + ), + ) + known = {arm.profile_id for arm in arms} + for assignment in sorted(frame.assignments, key=lambda item: item.profile_id): + if assignment.profile_id in known: + continue + known.add(assignment.profile_id) + arms.append( + ArmRow( + profile_id=assignment.profile_id, + name=assignment.name, + model=assignment.model, + framework_version=assignment.framework_version, + selection_order=None, + max_context_tokens=assignment.max_context_tokens, + ) + ) + return arms + + +def _participant_arm( + assignment: Optional[AssignmentRow], arms_by_id: Mapping[str, ArmRow] +) -> Optional[dict[str, Any]]: + if assignment is None: + return None + arm = arms_by_id.get(assignment.profile_id) + + def label(own: Optional[str], attribute: str) -> Optional[str]: + if own is not None: + return own + return getattr(arm, attribute) if arm is not None else None + + return { + "profile_id": assignment.profile_id, + "name": label(assignment.name, "name"), + "model": label(assignment.model, "model"), + "framework_version": label(assignment.framework_version, "framework_version"), + "assignment_status": assignment.status, + "assigned_at": iso(assignment.assigned_at), + "strategy": assignment.strategy, + } + + +def health( + status: str, + analysis: ParticipantAnalysis, + now: datetime, +) -> str: + if status != ACTIVE_ENROLLMENT_STATUS: + return "INACTIVE" + if not analysis.has_telemetry: + return "NO_TELEMETRY" + last = as_utc(analysis.last_event_at) + if last is not None and as_utc(now) - last <= ACTIVE_WINDOW: # type: ignore[operator] + return "ACTIVE" + return "IDLE" + + +def _latest(values: Iterable[Optional[datetime]]) -> Optional[datetime]: + present = [as_utc(value) for value in values if value is not None] + return max(present) if present else None # type: ignore[type-var] + + +def participant_header( + enrollment: EnrollmentRow, + assignment: Optional[AssignmentRow], + arms_by_id: Mapping[str, ArmRow], + sessions: Sequence[SessionRow], + analysis: ParticipantAnalysis, + now: datetime, + budget: Optional[Mapping[str, Any]] = None, +) -> dict[str, Any]: + """One participants-table row (study-local identity only).""" + return { + "enrollment_id": enrollment.enrollment_id, + # Metered budget (shared provider key): ``None`` for unmetered arms. + "budget": dict(budget) if budget is not None else None, + "participant_code": enrollment.participant_code, + "status": enrollment.status, + "enrolled_at": iso(enrollment.enrolled_at), + "consent_accepted_at": iso(enrollment.consent_accepted_at), + "consent_digest": enrollment.consent_digest, + "consent_answers": enrollment.consent_answers, + "arm": _participant_arm(assignment, arms_by_id), + "sessions": { + "total": len(sessions), + "active": sum( + 1 for session in sessions if session.state not in TERMINAL_SESSION_STATES + ), + "session_seconds": _r1(analysis.session_seconds), + "last_activity_at": iso(_latest(session.last_activity_at for session in sessions)), + "last_heartbeat_at": iso( + _latest(session.last_heartbeat_at for session in sessions) + ), + }, + "activity": { + "prompts": len(analysis.prompts), + "tool_calls": len(analysis.tool_calls), + "tool_failures": analysis.tool_failures, + "cancellations": len(analysis.cancels), + "permission_requests": len(analysis.permission_requests), + "permission_denials": analysis.decision_counts.get("reject", 0), + "errors": len(analysis.errors), + "agent_file_writes": analysis.agent_file_writes, + "ide_edits": analysis.ide_edits, + "usage_tokens": analysis.usage_tokens, + "active_days": analysis.active_days, + "first_event_at": iso(analysis.first_event_at), + "last_event_at": iso(analysis.last_event_at), + }, + "health": health(enrollment.status, analysis, now), + } + + +def _group(rows: Iterable[Any], attribute: str) -> dict[Any, list[Any]]: + grouped: dict[Any, list[Any]] = defaultdict(list) + for row in rows: + grouped[getattr(row, attribute)].append(row) + return grouped + + +def _enrollment_sort_key(enrollment: EnrollmentRow) -> tuple: + return (as_utc(enrollment.enrolled_at) or _UTC_MAX, enrollment.participant_code) + + +# -- endpoint A: participants table -------------------------------------------------- + + +def build_participants( + frame: StudyFrame, + events: Iterable[EventRow], + daily: Iterable[DailyEventCount], + *, + study_id: str, + now: datetime, + budgets: Optional[Mapping[str, Mapping[str, Any]]] = None, +) -> dict[str, Any]: + events_by_enrollment = _group(events, "enrollment_id") + daily_by_enrollment = _group(daily, "enrollment_id") + sessions_by_enrollment = _group(frame.sessions, "enrollment_id") + assignment_by_enrollment = { + assignment.enrollment_id: assignment for assignment in frame.assignments + } + arms = arm_catalogue(frame) + arms_by_id = {arm.profile_id: arm for arm in arms} + arm_members = Counter(assignment.profile_id for assignment in frame.assignments) + + participants = [] + for enrollment in sorted(frame.enrollments, key=_enrollment_sort_key): + sessions = sessions_by_enrollment.get(enrollment.enrollment_id, []) + analysis = analyze_participant( + events_by_enrollment.get(enrollment.enrollment_id, ()), + sessions, + daily_by_enrollment.get(enrollment.enrollment_id) or None, + ) + participants.append( + participant_header( + enrollment, + assignment_by_enrollment.get(enrollment.enrollment_id), + arms_by_id, + sessions, + analysis, + now, + budget=(budgets or {}).get(str(enrollment.enrollment_id)), + ) + ) + return { + "study_id": study_id, + "generated_at": iso(now), + "arms": [ + { + **_arm_labels(arm), + "selection_order": arm.selection_order, + "participants": arm_members.get(arm.profile_id, 0), + } + for arm in arms + ], + "participants": participants, + } + + +# -- endpoint B: one participant's dashboard ------------------------------------------- + + +def _timeline_rows( + timeline: Iterable[EventRow], analysis: ParticipantAnalysis +) -> list[dict[str, Any]]: + relay_calls = { + (call.session_id, call.tool_call_id): call + for call in analysis.call_lookup.values() + if call.is_relay and call.tool_call_id + } + items = [ + row + for row in timeline + if not is_message_chunk(row) + and row.event_type not in TIMELINE_EXCLUDED_EVENT_TYPES + and ( + row.event_type not in PERMISSION_EVENTS + or analysis.permission_filter.counts(row) + ) + ] + items.sort(key=order_key, reverse=True) + output = [] + for row in items[:MAX_TIMELINE]: + tool_name, tool_kind = display_tool_name(row.tool_name), row.tool_kind + if row.event_type in TOOL_EVENTS or row.event_type.startswith("permission."): + call = None + if row.tool_call_id: + call = ( + relay_calls.get((row.session_id, row.tool_call_id)) + if row.source == RELAY_SOURCE + else analysis.call_lookup.get( + (False, row.session_id, row.emitter_id, row.tool_call_id) + ) + ) + if call is not None: + # The call's name may come from the relay (a row's own title is + # never shown). + tool_name = tool_name or display_tool_name(call.tool_name) + tool_kind = tool_kind or call.tool_kind + output.append( + { + "occurred_at": iso(row.occurred_at), + "event_type": row.event_type, + "source": row.source, + "session_id": row.session_id, + "turn_id": row.turn_id, + "tool_name": tool_name, + "tool_kind": tool_kind, + "status": row.status or row.lifecycle_state, + "decision": ( + permission_decision(row) + if row.event_type == PERMISSION_DECIDED_EVENT + else row.decision + ), + "stop_reason": row.stop_reason, + "error_code": row.error_code, + # Chat lifecycle (which ACP method; why a chat's process ended). + "acp_method": row.acp_method, + "end_reason": row.end_reason, + } + ) + return output + + +def build_participant_detail( + frame: StudyFrame, + enrollment_id: str, + events: Iterable[EventRow], + daily: Iterable[DailyEventCount], + timeline: Iterable[EventRow], + *, + study_id: str, + now: datetime, +) -> Optional[dict[str, Any]]: + """The per-participant dashboard, or ``None`` for an unknown enrollment.""" + enrollment = next( + (row for row in frame.enrollments if row.enrollment_id == enrollment_id), None + ) + if enrollment is None: + return None + sessions = [row for row in frame.sessions if row.enrollment_id == enrollment_id] + assignment = next( + (row for row in frame.assignments if row.enrollment_id == enrollment_id), None + ) + arms_by_id = {arm.profile_id: arm for arm in arm_catalogue(frame)} + analysis = analyze_participant( + [row for row in events if row.enrollment_id in (None, enrollment_id)], + sessions, + [row for row in daily if row.enrollment_id == enrollment_id] or None, + ) + + prompts_by_day = Counter(_utc_day(prompt.occurred_at) for prompt in analysis.prompts) + tools_by_day = Counter(_utc_day(call.first_at) for call in analysis.tool_calls) + errors_by_day = Counter(_utc_day(error.occurred_at) for error in analysis.errors) + days = ( + {day for day, count in analysis.events_by_day.items() if count} + | {day for day, seconds in analysis.session_seconds_by_day.items() if seconds > 0} + | set(prompts_by_day) + | set(tools_by_day) + | set(errors_by_day) + | {day for day, count in analysis.ide_edits_by_day.items() if count} + ) + daily_rows = [ + { + "date": day.isoformat(), + "prompts": prompts_by_day.get(day, 0), + "tool_calls": tools_by_day.get(day, 0), + "errors": errors_by_day.get(day, 0), + "ide_edits": analysis.ide_edits_by_day.get(day, 0), + "session_seconds": _r1(analysis.session_seconds_by_day.get(day, 0.0)), + } + for day in sorted(days) + ] + + prompts_by_session = Counter(prompt.session_id for prompt in analysis.prompts) + tools_by_session = Counter(call.session_id for call in analysis.tool_calls) + errors_by_session = Counter(error.session_id for error in analysis.errors) + ordered_sessions = sorted( + sessions, + key=lambda row: as_utc(row.opened_at or row.created_at) or _UTC_MIN, + reverse=True, + ) + sessions_list = [ + { + "session_id": row.session_id, + "state": row.state, + "opened_at": iso(row.opened_at), + "closed_at": iso(row.closed_at), + "last_activity_at": iso(row.last_activity_at), + "close_reason": row.close_reason, + "session_seconds": _r1(analysis.session_seconds_by_session.get(row.session_id, 0.0)), + "prompts": prompts_by_session.get(row.session_id, 0), + "tool_calls": tools_by_session.get(row.session_id, 0), + "errors": errors_by_session.get(row.session_id, 0), + } + for row in ordered_sessions[:MAX_SESSIONS_LIST] + ] + + turns = [ + { + "turn_id": turn.turn_id, + "session_id": turn.session_id, + "started_at": iso(turn.started_at), + "completed_at": iso(turn.completed_at), + "duration_seconds": _r1(turn.duration_seconds), + "tool_calls": len(turn.tool_calls), + "tool_failures": turn.tool_failures, + "permission_requests": turn.permission_requests, + "stop_reason": turn.stop_reason, + "usage_tokens": turn.usage_tokens, + "cancelled": turn.cancelled, + } + for turn in sorted(analysis.turns, key=lambda turn: turn.start_order, reverse=True)[ + :MAX_TURNS + ] + ] + + cap_source: Optional[int] = None + if assignment is not None: + cap_source = assignment.max_context_tokens + if cap_source is None and assignment.profile_id in arms_by_id: + cap_source = arms_by_id[assignment.profile_id].max_context_tokens + + return { + "study_id": study_id, + "generated_at": iso(now), + **participant_header(enrollment, assignment, arms_by_id, sessions, analysis, now), + "metrics": analysis.metrics(), + "context": context_block(analysis.model_calls, effective_context_cap(cap_source)), + "daily": daily_rows, + "tool_kinds": tool_kind_rows(analysis.tool_calls), + "tools": tool_rows( + ((call, None) for call in analysis.tool_calls), limit=TOP_TOOLS_PARTICIPANT + ), + "stop_reasons": _counter_rows(analysis.stop_reason_counts, "stop_reason"), + "permission_decisions": _counter_rows(analysis.decision_counts, "decision"), + "sessions_list": sessions_list, + "turns": turns, + "timeline": _timeline_rows( + [row for row in timeline if row.enrollment_id in (None, enrollment_id)], + analysis, + ), + "chats": chat_rows(analysis.chats), + "chat_summary": chat_summary(analysis), + } + + +# -- endpoint C: study summary / arm comparison ------------------------------------------- + + +def _coverage_state(covered: int, total: int) -> str: + if total <= 0 or covered <= 0: + return "UNAVAILABLE" + if covered >= total: + return "AVAILABLE" + return "PARTIAL" + + +class FilterError(ValueError): + """An analytics filter value that names no arm or participant of the study.""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + + +def resolve_filters( + frame: StudyFrame, + arms: Optional[Sequence[str]] = None, + participants: Optional[Sequence[str]] = None, +) -> tuple[StudyFrame, Optional[frozenset[str]], dict[str, Any]]: + """Restrict a study frame to the selected arms and participants (AND). + + Returns the restricted frame (every arm is kept, so colours and order stay + stable), the enrollment ids event queries must be scoped to (``None`` when + nothing is filtered) and the applied filters to echo back. + """ + arm_ids = list(dict.fromkeys(str(item) for item in arms or ())) + participant_ids = list(dict.fromkeys(str(item) for item in participants or ())) + enrollments = {enrollment.enrollment_id: enrollment for enrollment in frame.enrollments} + known_arms = {arm.profile_id for arm in frame.arms} + unknown = [item for item in arm_ids if item not in known_arms] + [ + item for item in participant_ids if item not in enrollments + ] + if unknown: + raise FilterError("UNKNOWN_FILTER_VALUE", f"not part of this study: {', '.join(unknown)}") + if not arm_ids and not participant_ids: + return frame, None, { + "arms": [], + "participants": [], + "matched_participants": len(frame.enrollments), + "spend_scope": "study", + } + arm_of = {assignment.enrollment_id: assignment.profile_id for assignment in frame.assignments} + selected = frozenset( + enrollment_id + for enrollment_id in enrollments + if (not arm_ids or arm_of.get(enrollment_id) in arm_ids) + and (not participant_ids or enrollment_id in participant_ids) + ) + restricted = StudyFrame( + arms=frame.arms, + enrollments=tuple(row for row in frame.enrollments if row.enrollment_id in selected), + assignments=tuple(row for row in frame.assignments if row.enrollment_id in selected), + sessions=tuple(row for row in frame.sessions if row.enrollment_id in selected), + ) + return restricted, selected, { + "arms": arm_ids, + "participants": [ + {"enrollment_id": item, "participant_code": enrollments[item].participant_code} + for item in participant_ids + ], + "matched_participants": len(selected), + "spend_scope": "filtered", + } + + +def filter_options(frame: StudyFrame) -> dict[str, Any]: + """The arms and participants an analytics view can be filtered by.""" + arm_of = {assignment.enrollment_id: assignment.profile_id for assignment in frame.assignments} + members = Counter(arm_of.values()) + return { + "arms": [ + {"profile_id": arm.profile_id, "name": arm.name, "participants": members.get(arm.profile_id, 0)} + for arm in sorted(frame.arms, key=lambda arm: (arm.selection_order is None, arm.selection_order or 0)) + ], + "participants": [ + { + "enrollment_id": enrollment.enrollment_id, + "participant_code": enrollment.participant_code, + "profile_id": arm_of.get(enrollment.enrollment_id), + "status": enrollment.status, + } + for enrollment in sorted(frame.enrollments, key=_enrollment_sort_key) + ], + } + + +def build_study_summary( + frame: StudyFrame, + events: Iterable[EventRow], + daily: Iterable[DailyEventCount], + *, + study_id: str, + now: datetime, + window: Optional[DateWindow] = None, + spend: Optional[Mapping[str, Any]] = None, +) -> dict[str, Any]: + window = window or DateWindow() + events_by_enrollment = _group(events, "enrollment_id") + daily_by_enrollment = _group(daily, "enrollment_id") + sessions_by_enrollment = _group(frame.sessions, "enrollment_id") + assignment_by_enrollment = { + assignment.enrollment_id: assignment for assignment in frame.assignments + } + arms = arm_catalogue(frame) + arm_ids = [arm.profile_id for arm in arms] + + analyses: dict[str, ParticipantAnalysis] = {} + for enrollment in sorted(frame.enrollments, key=_enrollment_sort_key): + analyses[enrollment.enrollment_id] = analyze_participant( + events_by_enrollment.get(enrollment.enrollment_id, ()), + sessions_by_enrollment.get(enrollment.enrollment_id, ()), + daily_by_enrollment.get(enrollment.enrollment_id) or None, + window=window, + ) + arm_of = { + enrollment_id: assignment.profile_id + for enrollment_id, assignment in assignment_by_enrollment.items() + if enrollment_id in analyses + } + metrics_of = {enrollment_id: analysis.metrics() for enrollment_id, analysis in analyses.items()} + windowed_sessions = [ + session for session in frame.sessions if session_opened_in_window(session, window) + ] + + # Totals over every enrollment of the study. + all_turns = [turn for analysis in analyses.values() for turn in analysis.turns] + completed_turns = [turn for turn in all_turns if turn.completed] + turns_with_usage = [turn for turn in completed_turns if turn.usage_tokens is not None] + decisions: Counter = Counter() + stop_reasons: Counter = Counter() + usage_values = [ + analysis.usage_tokens + for analysis in analyses.values() + if analysis.usage_tokens is not None + ] + for analysis in analyses.values(): + decisions.update(analysis.decision_counts) + stop_reasons.update(analysis.stop_reason_counts) + usage_total = sum(usage_values) if usage_values else None + usage_coverage = _ratio(len(turns_with_usage), len(completed_turns)) + all_calls = [ + (call, arm_of.get(enrollment_id)) + for enrollment_id, analysis in analyses.items() + for call in analysis.tool_calls + ] + totals = { + "participants_enrolled": len(frame.enrollments), + "participants_active": sum( + 1 + for enrollment in frame.enrollments + if enrollment.status == ACTIVE_ENROLLMENT_STATUS + ), + "participants_with_telemetry": sum( + 1 for analysis in analyses.values() if analysis.has_telemetry + ), + "sessions": len(windowed_sessions), + "session_seconds": _r1(sum(analysis.session_seconds for analysis in analyses.values())), + "prompts": sum(len(analysis.prompts) for analysis in analyses.values()), + "tool_calls": len(all_calls), + "tool_failures": sum(1 for call, _ in all_calls if call.failed), + "cancellations": sum(len(analysis.cancels) for analysis in analyses.values()), + "permission_requests": sum( + len(analysis.permission_requests) for analysis in analyses.values() + ), + "permission_allowed": decisions.get("allow", 0), + "permission_rejected": decisions.get("reject", 0), + "permission_cancelled": decisions.get("cancelled", 0), + "errors": sum(len(analysis.errors) for analysis in analyses.values()), + "agent_file_writes": sum( + analysis.agent_file_writes for analysis in analyses.values() + ), + "ide_edits": sum(analysis.ide_edits for analysis in analyses.values()), + "usage_tokens": usage_total, + "usage_coverage": _r3(usage_coverage), + } + + # Arms: participant-level aggregation (the randomisation unit). + arms_out = [] + for arm in arms: + members = sorted( + enrollment_id + for enrollment_id, profile_id in arm_of.items() + if profile_id == arm.profile_id + ) + with_telemetry = [ + enrollment_id for enrollment_id in members if analyses[enrollment_id].has_telemetry + ] + metric_block = {} + for key in METRIC_KEYS: + population = members if key in COUNT_METRIC_KEYS else with_telemetry + metric_block[key] = metric_summary( + metrics_of[enrollment_id][key] for enrollment_id in population + ) + arm_decisions: Counter = Counter() + arm_stops: Counter = Counter() + arm_turns: list[Turn] = [] + arm_calls: list[ToolCall] = [] + arm_model_calls: list[EventRow] = [] + for enrollment_id in members: + analysis = analyses[enrollment_id] + arm_decisions.update(analysis.decision_counts) + arm_stops.update(analysis.stop_reason_counts) + arm_turns.extend(analysis.turns) + arm_calls.extend(analysis.tool_calls) + arm_model_calls.extend(analysis.model_calls) + turn_seconds = [ + turn.duration_seconds for turn in arm_turns if turn.duration_seconds is not None + ] + arms_out.append( + { + **_arm_labels(arm), + "selection_order": arm.selection_order, + "participants": len(members), + "participants_with_telemetry": len(with_telemetry), + "metrics": metric_block, + "stop_reasons": _counter_rows(arm_stops, "stop_reason"), + "permission_decisions": _counter_rows(arm_decisions, "decision"), + "tool_kinds": tool_kind_rows(arm_calls), + "turn_seconds": { + "p50": _r1(percentile(turn_seconds, 0.5)), + "p90": _r1(percentile(turn_seconds, 0.9)), + "n": len(turn_seconds), + }, + "transitions": transitions(arm_turns), + "context": context_block( + arm_model_calls, effective_context_cap(arm.max_context_tokens) + ), + } + ) + + # Daily activity (UTC dates with any activity). + active_by_day: dict[date, set[str]] = defaultdict(set) + prompts_by_day: Counter = Counter() + tools_by_day: Counter = Counter() + errors_by_day: Counter = Counter() + sessions_by_day: Counter = Counter() + arm_prompts: Counter = Counter() + arm_active: dict[tuple[date, str], set[str]] = defaultdict(set) + for enrollment_id, analysis in analyses.items(): + arm_id = arm_of.get(enrollment_id) + for day, count in analysis.events_by_day.items(): + if count: + active_by_day[day].add(enrollment_id) + if arm_id is not None: + arm_active[(day, arm_id)].add(enrollment_id) + for prompt in analysis.prompts: + day = _utc_day(prompt.occurred_at) + prompts_by_day[day] += 1 + if arm_id is not None: + arm_prompts[(day, arm_id)] += 1 + for call in analysis.tool_calls: + tools_by_day[_utc_day(call.first_at)] += 1 + for error in analysis.errors: + errors_by_day[_utc_day(error.occurred_at)] += 1 + for session in windowed_sessions: + if session.opened_at is not None: + sessions_by_day[_utc_day(session.opened_at)] += 1 + days = ( + set(active_by_day) + | set(prompts_by_day) + | set(tools_by_day) + | set(errors_by_day) + | set(sessions_by_day) + ) + daily_rows = [ + { + "date": day.isoformat(), + "active_participants": len(active_by_day.get(day, ())), + "prompts": prompts_by_day.get(day, 0), + "tool_calls": tools_by_day.get(day, 0), + "errors": errors_by_day.get(day, 0), + "sessions": sessions_by_day.get(day, 0), + "by_arm": { + arm_id: { + "prompts": arm_prompts.get((day, arm_id), 0), + "active_participants": len(arm_active.get((day, arm_id), ())), + } + for arm_id in arm_ids + }, + } + for day in sorted(days) + ] + + # Coverage of the fields the arm comparison depends on. + prompts_total = [prompt for analysis in analyses.values() for prompt in analysis.prompts] + if usage_coverage is not None: + usage_state = _coverage_state(len(turns_with_usage), len(completed_turns)) + if usage_state == "UNAVAILABLE" and usage_total is not None: + usage_state = "PARTIAL" + else: + usage_state = "PARTIAL" if usage_total is not None else "UNAVAILABLE" + coverage = { + "usage_tokens": usage_state, + "turn_correlation": _coverage_state( + sum(1 for prompt in prompts_total if prompt.turn_id is not None), + len(prompts_total), + ), + "tool_kind": _coverage_state( + sum(1 for call, _ in all_calls if call.tool_kind is not None), len(all_calls) + ), + } + + if spend is not None: + # Exact metered spend from the budget ledger (all time, not windowed). + totals["metered_spend_micro_usd"] = int(spend.get("metered_spend_micro_usd", 0)) + totals["metered_calls"] = int(spend.get("metered_calls", 0)) + totals["metered_reserved_micro_usd"] = int(spend.get("reserved_micro_usd", 0)) + + return { + "study_id": study_id, + "generated_at": iso(now), + "window": window.as_json(), + "totals": totals, + "arms": arms_out, + "daily": daily_rows, + "tool_kinds": tool_kind_rows(call for call, _ in all_calls), + "tools": tool_rows(all_calls, limit=TOP_TOOLS_STUDY, arm_ids=arm_ids), + "stop_reasons": _counter_rows(stop_reasons, "stop_reason"), + "permission_decisions": _counter_rows(decisions, "decision"), + "coverage": coverage, + "chat_history": chat_history(analyses, arm_of, arm_ids), + } diff --git a/src/research/analysis/study_analytics/models.py b/src/research/analysis/study_analytics/models.py new file mode 100644 index 0000000..fd2c138 --- /dev/null +++ b/src/research/analysis/study_analytics/models.py @@ -0,0 +1,197 @@ +"""Plain row types for the study analytics read models (issue 03). + +The store (:mod:`research.analysis.study_analytics.store`) loads only the +columns and JSONB metadata paths these rows carry, and the pure metric functions +(:mod:`research.analysis.study_analytics.metrics`) compute over them. No row +holds an ORM object, a whole telemetry envelope, a content field (prompt or +message text, tool arguments/results, reasoning, error messages) or any account +identity: participants are identified by their study-local enrollment id and +participant code only. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import date, datetime, time, timedelta, timezone +from typing import Optional, Sequence + +__all__ = [ + "ArmRow", + "AssignmentRow", + "DailyEventCount", + "DateWindow", + "EnrollmentRow", + "EventRow", + "SessionRow", + "StudyFrame", +] + + +@dataclass(frozen=True) +class ArmRow: + """One frozen study arm: ``study_agent_profile`` snapshot labels only.""" + + profile_id: str + name: Optional[str] = None + model: Optional[str] = None + framework_version: Optional[str] = None + selection_order: Optional[int] = None + max_context_tokens: Optional[int] = None + + +@dataclass(frozen=True) +class EnrollmentRow: + """One study-local enrollment (never joined to a login account).""" + + enrollment_id: str + participant_code: str + status: str + enrolled_at: Optional[datetime] = None + consent_accepted_at: Optional[datetime] = None + # The accepted consent version and the participant's answers per statement + # (both null for enrollments created before versions were recorded). + consent_digest: Optional[str] = None + consent_answers: Optional[dict[str, bool]] = None + + +@dataclass(frozen=True) +class AssignmentRow: + """One sticky assignment with the labels of its own frozen snapshot.""" + + enrollment_id: str + profile_id: str + status: str + assigned_at: Optional[datetime] = None + name: Optional[str] = None + model: Optional[str] = None + framework_version: Optional[str] = None + max_context_tokens: Optional[int] = None + # RANDOM_EQUAL / DETERMINISTIC_HASH, or MANUAL after an owner override. + strategy: Optional[str] = None + + +@dataclass(frozen=True) +class SessionRow: + """One research session's lifecycle timestamps.""" + + session_id: str + enrollment_id: str + state: str + opened_at: Optional[datetime] = None + closed_at: Optional[datetime] = None + last_activity_at: Optional[datetime] = None + last_heartbeat_at: Optional[datetime] = None + close_reason: Optional[str] = None + created_at: Optional[datetime] = None + + +@dataclass(frozen=True) +class EventRow: + """One metadata-only projection of a retained canonical event. + + ``tool_call_id`` is ``correlations.tool_call_id`` falling back to + ``payload.tool_call_id`` (the ACP normalizer only sets the payload key on + tool events). ``plan_completed`` is the ``completed`` entry of + ``payload.plan_status_counts`` (``None`` when the plan reported no status + counts). ``message_kind`` is set only on streamed message/thought chunks, + which are *not* prompts. ``prompt_tokens`` is ``metrics.counts.prompt_tokens`` + (provider-reported, carried by relay-source model-call events). + ``decision_scope`` is set on the built-in agent's own relay permission + reports (``policy`` or ``session_cached`` when no one was asked). + ``session_id`` is the *research* session; ``chat_id`` is the ACP chat + (``payload.session_id``, stamped by the participant proxy). ``acp_method``, + ``end_reason`` and ``selected_option_id`` follow + :mod:`research.telemetry.chat_lifecycle`. + """ + + event_type: str + source: str + occurred_at: datetime + enrollment_id: Optional[str] = None + session_id: Optional[str] = None + emitter_id: str = "" + emitter_sequence: int = 0 + turn_id: Optional[str] = None + tool_call_id: Optional[str] = None + permission_id: Optional[str] = None + tool_name: Optional[str] = None + tool_kind: Optional[str] = None + status: Optional[str] = None + lifecycle_state: Optional[str] = None + decision: Optional[str] = None + decision_scope: Optional[str] = None + stop_reason: Optional[str] = None + error_code: Optional[str] = None + message_kind: Optional[str] = None + usage_tokens: Optional[int] = None + prompt_tokens: Optional[int] = None + latency_ms: Optional[int] = None + plan_size: Optional[int] = None + plan_completed: Optional[int] = None + chat_id: Optional[str] = None + acp_method: Optional[str] = None + end_reason: Optional[str] = None + selected_option_id: Optional[str] = None + #: A ``permission.requested`` that offered the "Revise…" option. + offers_revise: bool = False + + +@dataclass(frozen=True) +class DailyEventCount: + """Per-enrollment, per-UTC-date presence over *all* retained events. + + ``ide_edits`` counts ``ide.document.changed`` events from the IDE collector: + one event is one document change (its ``payload.count`` is the number of + inserted characters, not a number of edits). + """ + + enrollment_id: str + day: date + events: int + ide_edits: int = 0 + first_at: Optional[datetime] = None + last_at: Optional[datetime] = None + + +@dataclass(frozen=True) +class DateWindow: + """An inclusive ``[start, end]`` range of UTC dates (either side open).""" + + start: Optional[date] = None + end: Optional[date] = None + + @property + def is_open(self) -> bool: + return self.start is None and self.end is None + + def bounds(self) -> tuple[Optional[datetime], Optional[datetime]]: + """Half-open UTC datetime bounds ``[lower, upper)`` of the window.""" + lower = ( + datetime.combine(self.start, time.min, tzinfo=timezone.utc) + if self.start is not None + else None + ) + # The last representable day has no next midnight: it leaves the + # window open above instead of overflowing. + upper = ( + datetime.combine(self.end + timedelta(days=1), time.min, tzinfo=timezone.utc) + if self.end is not None and self.end < date.max + else None + ) + return lower, upper + + def as_json(self) -> dict[str, Optional[str]]: + return { + "start": self.start.isoformat() if self.start is not None else None, + "end": self.end.isoformat() if self.end is not None else None, + } + + +@dataclass(frozen=True) +class StudyFrame: + """A study's frozen arms, enrollments, assignments and sessions.""" + + arms: Sequence[ArmRow] = field(default_factory=tuple) + enrollments: Sequence[EnrollmentRow] = field(default_factory=tuple) + assignments: Sequence[AssignmentRow] = field(default_factory=tuple) + sessions: Sequence[SessionRow] = field(default_factory=tuple) diff --git a/src/research/analysis/study_analytics/store.py b/src/research/analysis/study_analytics/store.py new file mode 100644 index 0000000..a9edbf7 --- /dev/null +++ b/src/research/analysis/study_analytics/store.py @@ -0,0 +1,564 @@ +"""Read-only SQL loading for the study analytics read models (issue 03). + +Every event query is study-scoped (``study_id``), excludes retention tombstones +(``retention_state = 'DELETED'``) and selects only scalar columns plus the +metadata JSONB paths the metric rows need (``envelope_json -> 'payload' ->> +'tool_kind'`` etc.): never a whole envelope and never a content field (prompt or +message text, tool arguments/results, reasoning, error messages). Nothing here +joins login identity (``research_participant``/``user``) and nothing writes. +""" + +from __future__ import annotations + +import math +import sys +from typing import TYPE_CHECKING, Any, Optional + +from sqlalchemy import String, and_, cast, func, literal_column, not_, or_, select, text + +from database.research_schemas import ( + ResearchEnrollment, + ResearchEvent, + StudyAgentProfile, + StudyAssignment, +) +from database.research_schemas import ResearchSessionV1 as ResearchSessionRow +from research.telemetry.chat_lifecycle import REVISE_OPTION_ID + +from .metrics import ( + ANALYTIC_EVENT_TYPES, + DOCUMENT_CHANGED_EVENT, + IDE_SOURCE, + MAX_TIMELINE, + PERMISSION_DECIDED_EVENT, + PROMPT_EVENT, + RELAY_SOURCE, + TIMELINE_EXCLUDED_EVENT_TYPES, + UNANSWERED_DECISION, + UNASKED_DECISION_SCOPES, +) +from .models import ( + ArmRow, + AssignmentRow, + DailyEventCount, + DateWindow, + EnrollmentRow, + EventRow, + SessionRow, + StudyFrame, +) + +if TYPE_CHECKING: + import uuid + from collections.abc import Collection + + from sqlalchemy.orm import Session + +__all__ = [ + "enrollment_study_id", + "load_daily_event_counts", + "load_events", + "load_study_frame", + "load_timeline", +] + +_ENVELOPE = ResearchEvent.envelope_json +_EVENT_BATCH_SIZE = 10_000 +_LIFECYCLE = _ENVELOPE["lifecycle_state"].astext +_DELETED = "DELETED" + + +def _payload(key: str): + return _ENVELOPE["payload"][key].astext + + +def _correlation(key: str): + return _ENVELOPE["correlations"][key].astext + + +def _metric(key: str): + return _ENVELOPE["metrics"][key].astext + + +def _metric_count(key: str): + return _ENVELOPE["metrics"]["counts"][key].astext + + +#: The metadata-only projection of one event (order matters: see ``_event_row``). +#: Ids are read as text: the rows only compare them as strings, and building +#: UUID objects for every row costs about a third of a large study's load. +_EVENT_COLUMNS = ( + cast(ResearchEvent.enrollment_id, String), + cast(ResearchEvent.research_session_id, String), + ResearchEvent.event_type, + ResearchEvent.source, + ResearchEvent.emitter_id, + ResearchEvent.emitter_sequence, + ResearchEvent.occurred_at, + _correlation("turn_id"), + func.coalesce(_correlation("tool_call_id"), _payload("tool_call_id")), + _correlation("permission_id"), + _payload("tool_name"), + _payload("tool_kind"), + _payload("status"), + _LIFECYCLE, + _payload("decision"), + _payload("decision_scope"), + _payload("stop_reason"), + _payload("error_code"), + _payload("message_kind"), + _metric("usage_tokens"), + _metric_count("prompt_tokens"), + _metric("latency_ms"), + _payload("plan_size"), + # A small list of {"status", "count"} records (plan entry text is never + # captured); null on every other event type. + _ENVELOPE["payload"]["plan_status_counts"], + # The ACP chat and its lifecycle (research.telemetry.chat_lifecycle). + _payload("session_id"), + _payload("acp_method"), + _payload("end_reason"), + _payload("selected_option_id"), + func.coalesce( + _ENVELOPE["payload"]["options"].contains([{"option_id": REVISE_OPTION_ID}]), False + ), +) + + +def _chunk_clause(): + """A streamed assistant/thought chunk (null-safe: always true or false).""" + return and_( + ResearchEvent.event_type == PROMPT_EVENT, + or_( + _payload("message_kind").isnot(None), + func.coalesce(_LIFECYCLE, "") == "started", + ), + ) + + +def _unasked_decision_clause(): + """A relay permission decision no one made (null-safe). + + The SQL form of :func:`metrics.relay_decision_made`. + """ + decision = _payload("decision") + return and_( + ResearchEvent.source == RELAY_SOURCE, + ResearchEvent.event_type == PERMISSION_DECIDED_EVENT, + or_( + decision.is_(None), + decision == UNANSWERED_DECISION, + func.coalesce(_payload("decision_scope"), "").in_( + sorted(UNASKED_DECISION_SCOPES) + ), + ), + ) + + +def _text(value: Any) -> Optional[str]: + return None if value is None else str(value) + + +def _int(value: Any) -> Optional[int]: + if value is None or isinstance(value, bool): + return None + if isinstance(value, int): + return value + try: + number = float(str(value).strip()) + except (TypeError, ValueError): + return None + if not math.isfinite(number) or number != int(number): + return None + return int(number) + + +def _plan_completed(value: Any) -> Optional[int]: + if not isinstance(value, list): + return None + completed = 0 + for entry in value: + if isinstance(entry, dict) and entry.get("status") == "completed": + completed += _int(entry.get("count")) or 0 + return completed + + +def _event_row(row: Any) -> EventRow: + ( + enrollment_id, + session_id, + event_type, + source, + emitter_id, + emitter_sequence, + occurred_at, + turn_id, + tool_call_id, + permission_id, + tool_name, + tool_kind, + status, + lifecycle_state, + decision, + decision_scope, + stop_reason, + error_code, + message_kind, + usage_tokens, + prompt_tokens, + latency_ms, + plan_size, + plan_status_counts, + chat_id, + acp_method, + end_reason, + selected_option_id, + offers_revise, + ) = tuple(row) + return EventRow( + event_type=event_type, + source=source, + occurred_at=occurred_at, + enrollment_id=_text(enrollment_id), + session_id=_text(session_id), + emitter_id=emitter_id or "", + emitter_sequence=int(emitter_sequence or 0), + turn_id=turn_id, + tool_call_id=tool_call_id, + permission_id=permission_id, + tool_name=tool_name, + tool_kind=tool_kind, + status=status, + lifecycle_state=lifecycle_state, + decision=decision, + decision_scope=decision_scope, + stop_reason=stop_reason, + error_code=error_code, + message_kind=message_kind, + usage_tokens=_int(usage_tokens), + prompt_tokens=_int(prompt_tokens), + latency_ms=_int(latency_ms), + plan_size=_int(plan_size), + plan_completed=_plan_completed(plan_status_counts), + # Chat ids repeat on every event of a chat: share one string per id. + chat_id=sys.intern(chat_id) if isinstance(chat_id, str) else None, + acp_method=acp_method, + end_reason=end_reason, + selected_option_id=selected_option_id, + offers_revise=bool(offers_revise), + ) + + +def _scoped( + statement, + study_id: uuid.UUID, + *, + enrollment_id: Optional[uuid.UUID] = None, + enrollment_ids: Optional[Collection[uuid.UUID]] = None, + window: Optional[DateWindow] = None, +): + statement = statement.where( + ResearchEvent.study_id == study_id, + ResearchEvent.retention_state != _DELETED, + ResearchEvent.enrollment_id.isnot(None), + ) + if enrollment_id is not None: + statement = statement.where(ResearchEvent.enrollment_id == enrollment_id) + if enrollment_ids is not None: + statement = statement.where(ResearchEvent.enrollment_id.in_(list(enrollment_ids))) + if window is not None: + lower, upper = window.bounds() + if lower is not None: + statement = statement.where(ResearchEvent.occurred_at >= lower) + if upper is not None: + statement = statement.where(ResearchEvent.occurred_at < upper) + return statement + + +def enrollment_study_id(session: Session, enrollment_id: uuid.UUID) -> Optional[uuid.UUID]: + """The study an enrollment belongs to, or ``None`` when it does not exist.""" + return session.execute( + select(ResearchEnrollment.study_id).where( + ResearchEnrollment.enrollment_id == enrollment_id + ) + ).scalar_one_or_none() + + +def load_study_frame( + session: Session, + study_id: uuid.UUID, + *, + enrollment_id: Optional[uuid.UUID] = None, +) -> StudyFrame: + """Arms, enrollments, assignments and sessions (no login identity).""" + arm_snapshot = StudyAgentProfile.profile_snapshot_json + arms = [ + ArmRow( + profile_id=str(profile_id), + name=name, + model=model, + framework_version=framework_version, + selection_order=selection_order, + max_context_tokens=_int(max_context_tokens), + ) + for ( + profile_id, + selection_order, + name, + model, + framework_version, + max_context_tokens, + ) in session.execute( + select( + StudyAgentProfile.profile_id, + StudyAgentProfile.selection_order, + arm_snapshot["name"].astext, + arm_snapshot["model"].astext, + arm_snapshot["framework_version"].astext, + arm_snapshot["max_context_tokens"].astext, + ) + .where(StudyAgentProfile.study_id == study_id) + .order_by(StudyAgentProfile.selection_order.asc()) + ).all() + ] + + enrollment_statement = select( + ResearchEnrollment.enrollment_id, + ResearchEnrollment.participant_code, + ResearchEnrollment.status, + ResearchEnrollment.enrolled_at, + ResearchEnrollment.consent_accepted_at, + ResearchEnrollment.consent_digest, + ResearchEnrollment.consent_snapshot_json["answers"], + ).where(ResearchEnrollment.study_id == study_id) + if enrollment_id is not None: + enrollment_statement = enrollment_statement.where( + ResearchEnrollment.enrollment_id == enrollment_id + ) + enrollments = [ + EnrollmentRow( + enrollment_id=str(row_id), + participant_code=participant_code, + status=str(status), + enrolled_at=enrolled_at, + consent_accepted_at=consent_accepted_at, + consent_digest=consent_digest, + consent_answers=consent_answers if isinstance(consent_answers, dict) else None, + ) + for ( + row_id, + participant_code, + status, + enrolled_at, + consent_accepted_at, + consent_digest, + consent_answers, + ) in session.execute(enrollment_statement).all() + ] + + assignment_snapshot = StudyAssignment.profile_snapshot_json + assignment_statement = ( + select( + StudyAssignment.enrollment_id, + StudyAssignment.agent_profile_id, + StudyAssignment.status, + StudyAssignment.assigned_at, + assignment_snapshot["name"].astext, + assignment_snapshot["model"].astext, + assignment_snapshot["framework_version"].astext, + assignment_snapshot["max_context_tokens"].astext, + StudyAssignment.strategy, + ) + .join( + ResearchEnrollment, + StudyAssignment.enrollment_id == ResearchEnrollment.enrollment_id, + ) + .where(ResearchEnrollment.study_id == study_id) + ) + if enrollment_id is not None: + assignment_statement = assignment_statement.where( + StudyAssignment.enrollment_id == enrollment_id + ) + assignments = [ + AssignmentRow( + enrollment_id=str(row_enrollment_id), + profile_id=str(profile_id), + status=str(status), + assigned_at=assigned_at, + name=name, + model=model, + framework_version=framework_version, + max_context_tokens=_int(max_context_tokens), + strategy=strategy, + ) + for ( + row_enrollment_id, + profile_id, + status, + assigned_at, + name, + model, + framework_version, + max_context_tokens, + strategy, + ) in session.execute(assignment_statement).all() + ] + + session_statement = ( + select( + ResearchSessionRow.session_id, + ResearchSessionRow.enrollment_id, + ResearchSessionRow.state, + ResearchSessionRow.opened_at, + ResearchSessionRow.closed_at, + ResearchSessionRow.last_activity_at, + ResearchSessionRow.last_heartbeat_at, + ResearchSessionRow.close_reason, + ResearchSessionRow.created_at, + ) + .join( + ResearchEnrollment, + ResearchSessionRow.enrollment_id == ResearchEnrollment.enrollment_id, + ) + .where(ResearchEnrollment.study_id == study_id) + ) + if enrollment_id is not None: + session_statement = session_statement.where( + ResearchSessionRow.enrollment_id == enrollment_id + ) + sessions = [ + SessionRow( + session_id=str(session_id), + enrollment_id=str(row_enrollment_id), + state=str(state), + opened_at=opened_at, + closed_at=closed_at, + last_activity_at=last_activity_at, + last_heartbeat_at=last_heartbeat_at, + close_reason=close_reason, + created_at=created_at, + ) + for ( + session_id, + row_enrollment_id, + state, + opened_at, + closed_at, + last_activity_at, + last_heartbeat_at, + close_reason, + created_at, + ) in session.execute(session_statement).all() + ] + return StudyFrame( + arms=arms, enrollments=enrollments, assignments=assignments, sessions=sessions + ) + + +def load_events( + session: Session, + study_id: uuid.UUID, + *, + enrollment_id: Optional[uuid.UUID] = None, + enrollment_ids: Optional[Collection[uuid.UUID]] = None, + window: Optional[DateWindow] = None, +) -> list[EventRow]: + """The metadata-only events the metric set reads (chunks excluded).""" + statement = _scoped( + select(*_EVENT_COLUMNS), + study_id, + enrollment_id=enrollment_id, + enrollment_ids=enrollment_ids, + window=window, + ).where( + ResearchEvent.event_type.in_(sorted(ANALYTIC_EVENT_TYPES)), + not_(_chunk_clause()), + ) + statement = statement.order_by( + ResearchEvent.occurred_at.asc(), + ResearchEvent.emitter_id.asc(), + ResearchEvent.emitter_sequence.asc(), + ) + # Streamed in batches: a large study has hundreds of thousands of rows, and + # buffering every driver row next to its EventRow doubles the memory peak. + # Streaming uses a server-side cursor, for which PostgreSQL favours + # fast-start plans (walking the global occurred_at index across every + # study); this query is always read to the end, so plan it as one. + session.execute(text("SET LOCAL cursor_tuple_fraction = 1.0")) + result = session.execute(statement.execution_options(yield_per=_EVENT_BATCH_SIZE)) + return [_event_row(row) for row in result] + + +def load_timeline( + session: Session, + study_id: uuid.UUID, + enrollment_id: uuid.UUID, + *, + limit: int = MAX_TIMELINE, +) -> list[EventRow]: + """The newest metadata-only events for a participant's replay timeline. + + Streamed message chunks, per-keystroke document changes, context-usage + updates and the built-in agent's reports of decisions no one made (one per + tool call under the auto policy) are left out; they would drown every other + event. + """ + statement = ( + _scoped(select(*_EVENT_COLUMNS), study_id, enrollment_id=enrollment_id) + .where( + ResearchEvent.event_type.notin_(sorted(TIMELINE_EXCLUDED_EVENT_TYPES)), + not_(_chunk_clause()), + not_(_unasked_decision_clause()), + ) + .order_by( + ResearchEvent.occurred_at.desc(), + ResearchEvent.emitter_id.desc(), + ResearchEvent.emitter_sequence.desc(), + ) + .limit(limit) + ) + return [_event_row(row) for row in session.execute(statement).all()] + + +def load_daily_event_counts( + session: Session, + study_id: uuid.UUID, + *, + enrollment_id: Optional[uuid.UUID] = None, + enrollment_ids: Optional[Collection[uuid.UUID]] = None, + window: Optional[DateWindow] = None, +) -> list[DailyEventCount]: + """Per-enrollment, per-UTC-date counts over every retained event.""" + day = func.date(func.timezone(literal_column("'UTC'"), ResearchEvent.occurred_at)) + statement = _scoped( + select( + ResearchEvent.enrollment_id, + day, + func.count(), + func.count().filter( + and_( + ResearchEvent.event_type == DOCUMENT_CHANGED_EVENT, + ResearchEvent.source == IDE_SOURCE, + ) + ), + func.min(ResearchEvent.occurred_at), + func.max(ResearchEvent.occurred_at), + ), + study_id, + enrollment_id=enrollment_id, + enrollment_ids=enrollment_ids, + window=window, + ).group_by(ResearchEvent.enrollment_id, day) + return [ + DailyEventCount( + enrollment_id=str(row_enrollment_id), + day=row_day, + events=int(events or 0), + ide_edits=int(ide_edits or 0), + first_at=first_at, + last_at=last_at, + ) + for row_enrollment_id, row_day, events, ide_edits, first_at, last_at in session.execute( + statement + ).all() + ] diff --git a/src/research/analysis/study_export/__init__.py b/src/research/analysis/study_export/__init__.py new file mode 100644 index 0000000..9b0e213 --- /dev/null +++ b/src/research/analysis/study_export/__init__.py @@ -0,0 +1,8 @@ +"""Owner-only raw data export of one study as a ZIP of CSV/JSONL files. + +Kept apart from the analytics read model: an export can include stored content +(only for a study that captured it, and only when asked), while analytics +responses never do. Participants appear only by enrollment id and study-local +participant code; account identity is never joined, and retention tombstones +are never exported. +""" diff --git a/src/research/analysis/study_export/build.py b/src/research/analysis/study_export/build.py new file mode 100644 index 0000000..ed615c3 --- /dev/null +++ b/src/research/analysis/study_export/build.py @@ -0,0 +1,333 @@ +"""Build a study export ZIP: one file per dataset plus ``manifest.json``. + +Every dataset is described by its column specs, which also become the +manifest's data dictionary. CSV cells are guarded against spreadsheet formula +injection; JSONL events carry the stored canonical envelope, from which account +linking keys are always removed and content is removed unless asked for. +""" + +from __future__ import annotations + +import csv +import io +import json +import tempfile +import zipfile +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import date, datetime +from typing import Any, Optional + +from research.analysis.study_analytics.metrics import ( + METRIC_KEYS, + display_tool_name, + permission_decision, +) +from research.analysis.study_analytics.models import EventRow +from research.telemetry.enums import FieldClass +from research.telemetry.privacy.engine import PrivacyPolicy, filter_payload + +__all__ = [ + "CHAT_COLUMNS", + "DATASETS", + "EVENT_CSV_COLUMNS", + "EVENT_JSONL_COLUMNS", + "FORMATS", + "PARTICIPANT_COLUMNS", + "SESSION_COLUMNS", + "Column", + "ExportWriter", + "csv_cell", + "event_csv_row", + "export_envelope", + "metric_columns", +] + +DATASETS = ("participants", "participant_metrics", "sessions", "chats", "events") +FORMATS = ("csv", "jsonl") +SPOOL_MAX_BYTES = 32 * 1024 * 1024 + +#: Keys that could link a record to an account; never exported. +LINKABLE_KEYS = ("account_id", "email", "session_token") +#: Spreadsheet formula triggers: a text cell starting with one is prefixed with '. +_FORMULA_PREFIXES = ("=", "+", "-", "@", "\t", "\r") + +#: Re-filtering stored payloads for an export without content: metadata is +#: kept as stored, every content-class value becomes ``[REDACTED]``. +_WITHOUT_CONTENT = PrivacyPolicy( + allowed_field_classes=[FieldClass.SYSTEM, FieldClass.BEHAVIORAL, FieldClass.CODE_METADATA], + content_allowed=False, + consent_active=True, + code_metadata_mode="allow", +) + + +@dataclass(frozen=True) +class Column: + name: str + description: str + + +PARTICIPANT_COLUMNS = ( + Column("enrollment_id", "Study-local enrollment id (the randomisation unit)."), + Column("participant_code", "Study-local pseudonym."), + Column("status", "Enrollment status (ACTIVE, REVOKED, WITHDRAWN, COMPLETED…)."), + Column("enrolled_at", "When the participant joined (UTC)."), + Column("consent_accepted_at", "When consent was accepted (UTC)."), + Column("consent_digest", "Version digest of the consent view accepted (null before versions were recorded)."), + Column("consent_answers", "Ticked consent statements as JSON {statement id: true/false}."), + Column("arm_profile_id", "Assigned arm (agent profile id)."), + Column("arm_name", "Assigned arm name."), + Column("arm_model", "Model of the assigned arm."), + Column("arm_runtime", "Runtime of the assigned arm."), + Column("assignment_strategy", "DETERMINISTIC_HASH or RANDOM_EQUAL (the draw), MANUAL (set by the owner before first use)."), + Column("assigned_at", "When the arm was assigned (UTC)."), + Column( + "randomized_profile_id", + "Arm the salted-hash draw gives this enrollment (studies with DETERMINISTIC_HASH; recomputable, " + "see manifest.assignment); differs from arm_profile_id only after a manual override.", + ), +) + +SESSION_COLUMNS = ( + Column("session_id", "Research session id (one IDE project activity boundary)."), + Column("enrollment_id", "Enrollment of the session."), + Column("participant_code", "Study-local pseudonym."), + Column("state", "Final or current session state."), + Column("opened_at", "When the session opened (UTC)."), + Column("closed_at", "When it closed (UTC), if it did."), + Column("last_activity_at", "Last explicit activity (UTC); a close the server applies stamps its own time here."), + Column("last_heartbeat_at", "Last heartbeat from the IDE (UTC): the session was still running then."), + Column("close_reason", "Why it closed."), + Column( + "session_seconds", + "This session's time as the analytics count it, within the exported range: opened_at to closed_at, ending " + "at last_heartbeat_at at the latest when the server closed the session on its own (idle_timeout, " + "resume_grace_expired, revoked, STUDY_STOPPED), which it does only when it next hears from the participant. " + "A participant's session time (participant_metrics) merges sessions that overlap (two windows open at " + "once), so it can be less than the sum of their rows.", + ), +) + +CHAT_COLUMNS = ( + Column("enrollment_id", "Enrollment of the chat."), + Column("participant_code", "Study-local pseudonym."), + Column("arm_profile_id", "Assigned arm."), + Column("chat_id", "ACP chat (session) id."), + Column( + "ordinal", + "Chat number in the participant's history (within the exported range); empty for a chat without a prompt.", + ), + Column( + "used", + "Whether the participant sent a prompt in the chat. IntelliJ also starts an agent for a chat it only creates " + "or shows (a new empty chat, the chat it switches to after a delete); those are not numbered or counted.", + ), + Column("started_at", "First start or event of the chat (UTC)."), + Column("ended_at", "When the chat ended (UTC), if observed."), + Column("start_kind", "new, fork, load (reopened), resume, or unknown (older telemetry)."), + Column( + "end_reason", + "close (the IDE closed the chat), host_closed / signal_terminated (the IDE ended the agent process: the " + "chat was deleted or the IDE closed), agent_exited, session_stale, open, or unknown.", + ), + Column("reopen_count", "Times the chat was reopened or resumed."), + Column("prompts", "Prompts in the chat."), + Column("tool_calls", "Tool calls in the chat."), + Column("cancels", "Turns interrupted by the participant."), + Column("rejections", "Approval requests rejected."), + Column("revisions", "Approval requests where the participant chose Revise… (even if the follow-up form was dismissed)."), + Column("usage_tokens", "Provider-reported tokens (when reported)."), + Column("previous_end_reason", "How the participant's previous chat ended."), + Column("gap_since_previous_seconds", "Seconds between the previous chat's end and this chat's start."), +) + +EVENT_CSV_COLUMNS = ( + Column("event_id", "Canonical event id."), + Column("occurred_at", "When the event happened (UTC)."), + Column("enrollment_id", "Enrollment of the event."), + Column("participant_code", "Study-local pseudonym."), + Column("arm_profile_id", "Assigned arm."), + Column("research_session_id", "Research session."), + Column("chat_id", "ACP chat (session) id, when the event belongs to one."), + Column("event_type", "Canonical event type."), + Column("source", "acp (proxy), relay (built-in agent), ide or server."), + Column("emitter_id", "Emitting process."), + Column("emitter_sequence", "Per-emitter sequence number (orders events of one emitter)."), + Column("lifecycle_state", "Lifecycle phase (started, completed, cancelled…)."), + Column("turn_id", "Prompt turn within the emitter."), + Column("tool_call_id", "Tool call id."), + Column("permission_id", "Approval request id."), + Column("message_id", "Streamed message id."), + Column("message_kind", "user (prompt text), assistant or thought (streamed chunks); empty on turn starts."), + Column("tool_name", "Tool identifier (titles that describe a command or file are left out)."), + Column("tool_kind", "ACP tool kind (read, edit, execute…)."), + Column("status", "Tool status."), + Column("decision", "Approval decision: allow, reject, revise (Revise… chosen), cancelled…"), + Column("selected_option_id", "Approval option picked."), + Column("stop_reason", "Why a turn ended."), + Column("error_code", "Error code."), + Column("acp_method", "ACP method the event was derived from."), + Column("end_reason", "Why a chat's agent process ended."), + Column("usage_tokens", "Provider-reported tokens."), + Column("latency_ms", "Reported latency (ms)."), +) + +EVENT_JSONL_COLUMNS = ( + Column("event_id", "Canonical event id."), + Column("occurred_at", "When the event happened (UTC)."), + Column("enrollment_id", "Enrollment of the event."), + Column("participant_code", "Study-local pseudonym."), + Column("arm_profile_id", "Assigned arm."), + Column("research_session_id", "Research session."), + Column("event_type", "Canonical event type."), + Column("source", "Event source."), + Column( + "envelope", + "The stored canonical envelope (payload, correlations, metrics, provenance). Content-class values are " + "[REDACTED] unless the export includes content; account-linking keys are always removed.", + ), +) + +_METRIC_HELP = "Participant-level metric (see the study analytics definitions)." + + +def metric_columns() -> tuple[Column, ...]: + return ( + Column("enrollment_id", "Enrollment."), + Column("participant_code", "Study-local pseudonym."), + Column("arm_profile_id", "Assigned arm."), + Column("has_telemetry", "Whether any event was retained for the participant in range."), + *(Column(key, _METRIC_HELP) for key in METRIC_KEYS), + ) + + +def _json_default(value: Any) -> Any: + if isinstance(value, (datetime, date)): + return value.isoformat() + return str(value) + + +def csv_cell(value: Any) -> str: + """One CSV cell; text that a spreadsheet would run as a formula is quoted.""" + if value is None: + return "" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, (dict, list)): + value = json.dumps(value, ensure_ascii=False, sort_keys=True, default=_json_default) + text = str(value) + return f"'{text}" if text.startswith(_FORMULA_PREFIXES) else text + + +def _without_linkable(mapping: Any) -> dict[str, Any]: + return {key: value for key, value in dict(mapping or {}).items() if key not in LINKABLE_KEYS} + + +def export_envelope(envelope: Mapping[str, Any], *, include_content: bool) -> dict[str, Any]: + """The stored envelope for an export: never account-linking keys, and + content only when asked for.""" + exported = dict(envelope) + payload = _without_linkable(exported.get("payload")) + if not include_content: + payload, summary = filter_payload(payload, _WITHOUT_CONTENT) + if summary.blocked: + payload = {} + exported["payload"] = payload + exported["provenance"] = _without_linkable(exported.get("provenance")) + return exported + + +def event_csv_row(envelope: Mapping[str, Any]) -> dict[str, Any]: + """The metadata projection of one envelope (no content-class values).""" + payload = envelope.get("payload") if isinstance(envelope.get("payload"), Mapping) else {} + correlations = envelope.get("correlations") if isinstance(envelope.get("correlations"), Mapping) else {} + metrics = envelope.get("metrics") if isinstance(envelope.get("metrics"), Mapping) else {} + decision_row = EventRow( + event_type=str(envelope.get("event_type") or ""), + source=str(envelope.get("source") or ""), + occurred_at=datetime.min, + decision=payload.get("decision"), + selected_option_id=payload.get("selected_option_id"), + ) + return { + "chat_id": payload.get("session_id"), + "lifecycle_state": envelope.get("lifecycle_state"), + "turn_id": correlations.get("turn_id"), + "tool_call_id": correlations.get("tool_call_id") or payload.get("tool_call_id"), + "permission_id": correlations.get("permission_id"), + "message_id": correlations.get("message_id") or payload.get("message_id"), + "message_kind": payload.get("message_kind"), + "tool_name": display_tool_name(payload.get("tool_name")), + "tool_kind": payload.get("tool_kind"), + "status": payload.get("status"), + "decision": permission_decision(decision_row) if payload.get("decision") is not None else None, + "selected_option_id": payload.get("selected_option_id"), + "stop_reason": payload.get("stop_reason"), + "error_code": payload.get("error_code"), + "acp_method": payload.get("acp_method"), + "end_reason": payload.get("end_reason"), + "usage_tokens": metrics.get("usage_tokens"), + "latency_ms": metrics.get("latency_ms"), + } + + +class ExportWriter: + """Writes datasets into a ZIP spooled to a temporary file (bounded memory).""" + + def __init__(self, export_format: str) -> None: + self.format = export_format + self.file = tempfile.SpooledTemporaryFile(max_size=SPOOL_MAX_BYTES) + self.zip = zipfile.ZipFile(self.file, "w", compression=zipfile.ZIP_DEFLATED) + self.counts: dict[str, int] = {} + + def write_rows(self, name: str, columns: Sequence[Column], rows: Iterable[Mapping[str, Any]]) -> int: + filename = f"{name}.{self.format}" + count = 0 + with self.zip.open(filename, "w", force_zip64=True) as raw: + stream = io.TextIOWrapper(raw, encoding="utf-8", newline="") + if self.format == "csv": + writer = csv.writer(stream) + writer.writerow([column.name for column in columns]) + for row in rows: + writer.writerow([csv_cell(row.get(column.name)) for column in columns]) + count += 1 + else: + for row in rows: + stream.write( + json.dumps( + {column.name: row.get(column.name) for column in columns}, + ensure_ascii=False, + default=_json_default, + ) + ) + stream.write("\n") + count += 1 + stream.flush() + stream.detach() + self.counts[filename] = count + return count + + def write_json(self, name: str, document: Any) -> None: + self.zip.writestr(name, json.dumps(document, indent=2, ensure_ascii=False, default=_json_default)) + + def finish(self) -> Any: + self.zip.close() + self.file.seek(0) + return self.file + + +def dictionary(columns_by_dataset: Mapping[str, Sequence[Column]]) -> dict[str, dict[str, str]]: + """The manifest's data dictionary: dataset -> column -> description.""" + return { + dataset: {column.name: column.description for column in columns} + for dataset, columns in columns_by_dataset.items() + } + + +def optional_iso(value: Optional[datetime]) -> Optional[str]: + return value.isoformat() if isinstance(value, datetime) else None diff --git a/src/research/analysis/study_export/store.py b/src/research/analysis/study_export/store.py new file mode 100644 index 0000000..da5d9da --- /dev/null +++ b/src/research/analysis/study_export/store.py @@ -0,0 +1,89 @@ +"""Stream a study's retained events for an export (no account identity).""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional + +from sqlalchemy import String, cast, or_, select + +from database.research_schemas import ResearchEvent + +if TYPE_CHECKING: + import uuid + from collections.abc import Collection, Iterator + + from sqlalchemy.orm import Session + + from research.analysis.study_analytics.models import DateWindow + +__all__ = ["EVENT_CATEGORIES", "iter_export_events"] + +_DELETED = "DELETED" +_BATCH_SIZE = 2_000 + +#: Export categories by canonical event type (prefix match; ``other`` is the rest). +EVENT_CATEGORIES: dict[str, tuple[str, ...]] = { + "conversation": ("agent.message.",), + "tools": ("tool.",), + "approvals": ("permission.",), + "chat_lifecycle": ("interaction.",), + "plans_usage": ("plan.", "usage."), + "ide": ("ide.",), + "errors": ("agent.error", "system."), +} + + +def _category_clause(categories: Collection[str]): + clauses = [] + known_prefixes = [prefix for prefixes in EVENT_CATEGORIES.values() for prefix in prefixes] + for category in categories: + if category == "other": + clauses.append(~or_(*(ResearchEvent.event_type.startswith(prefix) for prefix in known_prefixes))) + continue + clauses.extend(ResearchEvent.event_type.startswith(prefix) for prefix in EVENT_CATEGORIES[category]) + return or_(*clauses) + + +def iter_export_events( + session: Session, + study_id: uuid.UUID, + *, + enrollment_ids: Optional[Collection[uuid.UUID]] = None, + window: Optional[DateWindow] = None, + categories: Optional[Collection[str]] = None, +) -> Iterator[tuple[Any, ...]]: + """``(event_id, occurred_at, enrollment_id, research_session_id, event_type, + source, emitter_id, emitter_sequence, envelope)`` in deterministic order.""" + statement = select( + cast(ResearchEvent.event_id, String), + ResearchEvent.occurred_at, + cast(ResearchEvent.enrollment_id, String), + cast(ResearchEvent.research_session_id, String), + ResearchEvent.event_type, + ResearchEvent.source, + ResearchEvent.emitter_id, + ResearchEvent.emitter_sequence, + ResearchEvent.envelope_json, + ).where( + ResearchEvent.study_id == study_id, + ResearchEvent.retention_state != _DELETED, + ResearchEvent.enrollment_id.isnot(None), + ) + if enrollment_ids is not None: + statement = statement.where(ResearchEvent.enrollment_id.in_(list(enrollment_ids))) + if window is not None: + lower, upper = window.bounds() + if lower is not None: + statement = statement.where(ResearchEvent.occurred_at >= lower) + if upper is not None: + statement = statement.where(ResearchEvent.occurred_at < upper) + if categories: + statement = statement.where(_category_clause(categories)) + statement = statement.order_by( + ResearchEvent.occurred_at.asc(), + ResearchEvent.emitter_id.asc(), + ResearchEvent.emitter_sequence.asc(), + ResearchEvent.event_id.asc(), + ) + for row in session.execute(statement.execution_options(yield_per=_BATCH_SIZE)): + yield tuple(row) diff --git a/src/research/analysis/study_traces/__init__.py b/src/research/analysis/study_traces/__init__.py new file mode 100644 index 0000000..97ac2aa --- /dev/null +++ b/src/research/analysis/study_traces/__init__.py @@ -0,0 +1,8 @@ +"""Owner-only chat traces: what the agent was asked, thought, said and did. + +Kept apart from :mod:`research.analysis.study_analytics` on purpose: analytics +responses never carry content, while a trace returns the stored text (prompts, +reasoning, messages, tool arguments and results) exactly as the study's +telemetry policy and the participant's consent kept it, and ``[REDACTED]`` +markers where they did not. +""" diff --git a/src/research/analysis/study_traces/assemble.py b/src/research/analysis/study_traces/assemble.py new file mode 100644 index 0000000..e12e49f --- /dev/null +++ b/src/research/analysis/study_traces/assemble.py @@ -0,0 +1,460 @@ +"""Assemble one chat's events into turns of readable blocks (pure). + +A turn starts at a participant prompt and runs to the next one; whatever +precedes the first prompt (the chat opening) is a ``preamble``. Inside a turn: +streamed reasoning and message chunks merge into one block per message, the +lifecycle events of a tool call merge into one block, and its approval request +and answer attach to that tool call. Text is returned as stored; a value the +privacy policy removed is reported as not captured, never guessed. +""" + +from __future__ import annotations + +import difflib +import json +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import PurePosixPath +from typing import TYPE_CHECKING, Any, Optional +from urllib.parse import urlparse + +from research.telemetry.chat_lifecycle import ( + ACP_METHOD_KEY, + CANCEL_METHOD, + END_REASON_KEY, + REVISE_OPTION_ID, +) + +if TYPE_CHECKING: + from .store import TraceRow + +__all__ = ["MAX_TEXT_CHARS", "build_page", "content_text", "diff_blocks"] + +REDACTED = "[REDACTED]" +#: Longest text returned for one block (the rest is cut and flagged). +MAX_TEXT_CHARS = 20_000 +#: A page stops early after this many events even inside one turn. +MAX_PAGE_EVENTS = 20_000 + +_PROMPT = "agent.message.started" +_COMPLETION = "agent.message.completed" +_TOOL_EVENTS = {"tool.created", "tool.started", "tool.completed", "tool.failed"} +_ERROR_EVENTS = {"agent.error", "system.agent.crashed", "system.proxy.error"} + + +def _cap(text: Optional[str]) -> tuple[Optional[str], bool]: + if text is None or len(text) <= MAX_TEXT_CHARS: + return text, False + return text[:MAX_TEXT_CHARS], True + + +def content_text(value: Any) -> tuple[Optional[str], bool]: + """Readable text of a stored ACP value, and whether policy removed it.""" + if value is None: + return None, False + if isinstance(value, str): + return (None, True) if value == REDACTED else (value, False) + if isinstance(value, list): + parts: list[str] = [] + redacted = False + for item in value: + text, hidden = content_text(item) + redacted = redacted or hidden + if text: + parts.append(text) + return ("\n".join(parts) if parts else None), redacted + if isinstance(value, Mapping): + kind = value.get("type") + if kind == "text": + return content_text(value.get("text")) + if kind == "content": + return content_text(value.get("content")) + if kind == "resource_link": + return f"[file: {value.get('uri') or value.get('name') or 'resource'}]", False + if kind == "resource": + resource = value.get("resource") if isinstance(value.get("resource"), Mapping) else {} + label = f"[file: {resource.get('uri')}]" if resource.get("uri") else "[file]" + text, hidden = content_text(resource.get("text")) + return (f"{label}\n{text}" if text else label), hidden + if kind in ("image", "audio"): + return f"[{kind}]", False + if kind == "terminal": + return "[terminal output]", False + if kind == "diff": + return None, False + return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str), False + return str(value), False + + +def diff_blocks(value: Any) -> list[dict[str, Any]]: + """The file diffs a tool call carried, as unified diffs (capped).""" + items = value if isinstance(value, list) else [value] + diffs: list[dict[str, Any]] = [] + for item in items: + if not isinstance(item, Mapping) or item.get("type") != "diff": + continue + old, new = item.get("oldText"), item.get("newText") + if old == REDACTED or new == REDACTED: + diffs.append({"path": item.get("path"), "diff": None, "redacted": True}) + continue + lines = difflib.unified_diff( + str(old or "").splitlines(), + str(new or "").splitlines(), + fromfile="before", + tofile="after", + lineterm="", + ) + text, truncated = _cap("\n".join(lines)) + diffs.append({"path": item.get("path"), "diff": text, "truncated": truncated}) + return diffs + + +def _decision(payload: Mapping[str, Any]) -> Optional[str]: + if payload.get("selected_option_id") == REVISE_OPTION_ID: + return "revise" + decision = payload.get("decision") + return str(decision) if decision is not None else None + + +def _is_turn_start(row: TraceRow) -> bool: + return ( + row.event_type == _PROMPT + and row.payload.get("message_kind") is None + and (row.lifecycle_state or "") != "started" + ) + + +def _ms(start: Optional[datetime], end: Optional[datetime]) -> Optional[int]: + if start is None or end is None or end < start: + return None + return int((end - start).total_seconds() * 1000) + + +@dataclass +class _Turn: + index: int + kind: str + started_at: datetime + turn_id: Optional[str] = None + research_session_id: Optional[str] = None + completed_at: Optional[datetime] = None + stop_reason: Optional[str] = None + usage_tokens: Optional[int] = None + continues: bool = False + blocks: list[dict[str, Any]] = field(default_factory=list) + tools: dict[str, dict[str, Any]] = field(default_factory=dict) + permissions: dict[str, dict[str, Any]] = field(default_factory=dict) + + def block(self, kind: str, at: datetime, **fields: Any) -> dict[str, Any]: + entry = {"type": kind, "at": at, **fields} + self.blocks.append(entry) + return entry + + def serialize(self) -> dict[str, Any]: + blocks = [] + for block in self.blocks: + entry = {key: value for key, value in block.items() if not key.startswith("_")} + at = entry.pop("at") + entry["at"] = at.isoformat() + entry["offset_ms"] = _ms(self.started_at, at) or 0 + for moment in ("started_at", "completed_at", "requested_at", "decided_at"): + if isinstance(entry.get(moment), datetime): + entry[moment] = entry[moment].isoformat() + blocks.append(entry) + return { + "index": self.index, + "kind": self.kind, + "turn_id": self.turn_id, + "research_session_id": self.research_session_id, + "started_at": self.started_at.isoformat(), + "completed_at": self.completed_at.isoformat() if self.completed_at else None, + "duration_ms": _ms(self.started_at, self.completed_at), + "stop_reason": self.stop_reason, + "usage_tokens": self.usage_tokens, + "continues": self.continues, + "blocks": blocks, + } + + +def _text_fields(value: Any) -> dict[str, Any]: + text, redacted = content_text(value) + text, truncated = _cap(text) + return {"text": text, "redacted": redacted, "truncated": truncated} + + +def _typed_prompt(value: Any) -> Any: + """What the participant typed: the prompt's text blocks, without what the IDE + sent along (IntelliJ links the open file to every prompt).""" + if isinstance(value, list): + return [item for item in value if isinstance(item, Mapping) and item.get("type") == "text"] + return value + + +def _prompt_attachments(value: Any) -> list[dict[str, Any]]: + """Files and media sent with a prompt, by file name only: never the path, + which names the participant's home folder.""" + attachments: list[dict[str, Any]] = [] + for item in value if isinstance(value, list) else []: + kind = item.get("type") if isinstance(item, Mapping) else None + if kind == "resource_link": + ref = item.get("name") or item.get("uri") + elif kind == "resource": + resource = item.get("resource") + ref = resource.get("uri") if isinstance(resource, Mapping) else None + elif kind in ("image", "audio"): + ref = item.get("uri") + else: + continue + ref = str(ref or "") + name = PurePosixPath(urlparse(ref).path or ref).name + attachments.append({"name": name or str(kind), "type": kind}) + return attachments + + +def _revision_instructions(payload: Mapping[str, Any]) -> Optional[dict[str, Any]]: + """The participant's instructions from a relayed decision's content blob + (``payload``: the agent's own payload as JSON, or ``[REDACTED]``).""" + blob = payload.get("payload") + if blob is None: + return None + if blob == REDACTED: + return _text_fields(REDACTED) + try: + own = json.loads(blob) if isinstance(blob, str) else blob + except ValueError: + return None + own = own.get("payload") if isinstance(own, Mapping) else None + text = own.get("text") if isinstance(own, Mapping) else None + return _text_fields(text) if text is not None else None + + +def _apply_revision(turn: _Turn, row: TraceRow) -> None: + """Attach the built-in agent's report of a "Revise…" form to its tool call: + how the form was answered, which parts were kept and the participant's + instructions (content, so only where the study captured it).""" + payload = row.payload + call_id = row.correlations.get("tool_call_id") or payload.get("tool_call_id") + tool = turn.tools.get(call_id) if call_id else None + if tool is None: + return + permission = tool.get("permission") + if permission is None: + permission = tool["permission"] = {"permission_id": None, "requested_at": None, "options": []} + if not permission.get("decision"): + permission["decision"] = "revise" + counts = row.metrics.get("counts") + counts = counts if isinstance(counts, Mapping) else {} + instructions = _revision_instructions(payload) + if payload.get("payload") is None and payload.get("elicitation_action") == "accept": + # A sent form carries instructions (a required field): a report stored + # without any content blob is one whose study did not keep them. + instructions = _text_fields(REDACTED) + permission["revision"] = { + "form": payload.get("elicitation_action"), + "status": payload.get("revise_status"), + "hunks": counts.get("hunk_count"), + "kept_hunks": counts.get("kept_hunk_count"), + "instructions": instructions, + } + + +def _apply(turn: _Turn, row: TraceRow) -> None: + payload = row.payload + if row.source != "acp": + # Only the agent's own Revise reports are read besides the proxy's events. + _apply_revision(turn, row) + return + kind = payload.get("message_kind") + if row.event_type == _PROMPT and kind == "user": + prompt = next((block for block in turn.blocks if block["type"] == "prompt"), None) + if prompt is None or prompt.get("captured"): + prompt = turn.block("prompt", row.occurred_at) + value = payload.get("prompt") + prompt.update(_text_fields(_typed_prompt(value)), captured=True) + attachments = _prompt_attachments(value) + if attachments: + prompt["attachments"] = attachments + return + if row.event_type == _PROMPT and kind in ("thought", "assistant"): + block_type = "thought" if kind == "thought" else "message" + value = payload.get("reasoning") if kind == "thought" else payload.get("content") + message_id = payload.get("message_id") or row.correlations.get("message_id") + text, redacted = content_text(value) + last = turn.blocks[-1] if turn.blocks else None + if last is not None and last["type"] == block_type and last.get("_message_id") == message_id: + merged = (last.get("_raw") or "") + (text or "") + last["_raw"] = merged + last["text"], last["truncated"] = _cap(merged or None) + last["redacted"] = last["redacted"] or redacted + last["chunks"] += 1 + return + capped, truncated = _cap(text) + turn.block( + block_type, + row.occurred_at, + text=capped, + redacted=redacted, + truncated=truncated, + chunks=1, + _message_id=message_id, + _raw=text or "", + ) + return + if row.event_type in _TOOL_EVENTS: + call_id = payload.get("tool_call_id") or row.correlations.get("tool_call_id") or row.event_id + tool = turn.tools.get(call_id) + if tool is None: + tool = turn.tools[call_id] = turn.block( + "tool", + row.occurred_at, + tool_call_id=call_id, + title=None, + tool_kind=None, + status=None, + arguments=None, + result=None, + diffs=[], + started_at=row.occurred_at, + completed_at=None, + duration_ms=None, + permission=None, + ) + for source_key, target_key in (("tool_name", "title"), ("tool_kind", "tool_kind")): + if payload.get(source_key): + tool[target_key] = payload[source_key] + if row.event_type == "tool.failed": + tool["status"] = "failed" + elif payload.get("status"): + tool["status"] = payload["status"] + if "arguments" in payload: + tool["arguments"] = _text_fields(payload["arguments"]) + if "content" in payload: + tool["result"] = _text_fields(payload["content"]) + diffs = diff_blocks(payload["content"]) + if diffs: + tool["diffs"] = diffs + if row.event_type in ("tool.completed", "tool.failed"): + tool["completed_at"] = row.occurred_at + tool["duration_ms"] = _ms(tool["started_at"], row.occurred_at) + return + if row.event_type == "permission.requested": + permission = { + "permission_id": row.correlations.get("permission_id"), + "requested_at": row.occurred_at, + "options": [dict(option) for option in payload.get("options") or [] if isinstance(option, Mapping)], + "decision": None, + "selected_option_id": None, + "decided_at": None, + "wait_ms": None, + } + call_id = payload.get("tool_call_id") or row.correlations.get("tool_call_id") + tool = turn.tools.get(call_id) if call_id else None + if tool is not None: + tool["permission"] = permission + else: + turn.block("permission", row.occurred_at, permission=permission) + if permission["permission_id"] is not None: + turn.permissions[str(permission["permission_id"])] = permission + return + if row.event_type == "permission.decided": + permission_id = row.correlations.get("permission_id") + permission = turn.permissions.get(str(permission_id)) if permission_id is not None else None + if permission is None: + permission = {"permission_id": permission_id, "requested_at": None, "options": []} + turn.block("permission", row.occurred_at, permission=permission) + permission.update( + decision=_decision(payload), + selected_option_id=payload.get("selected_option_id"), + decided_at=row.occurred_at, + wait_ms=_ms(permission.get("requested_at"), row.occurred_at), + ) + return + if row.event_type == _COMPLETION: + turn.completed_at = row.occurred_at + turn.stop_reason = payload.get("stop_reason") + usage = row.metrics.get("usage_tokens") + if isinstance(usage, int) and not isinstance(usage, bool): + turn.usage_tokens = usage + return + if row.event_type == "plan.updated": + counts = payload.get("plan_status_counts") or [] + completed = sum( + int(entry.get("count") or 0) + for entry in counts + if isinstance(entry, Mapping) and entry.get("status") == "completed" + ) + turn.block("plan", row.occurred_at, plan_size=payload.get("plan_size"), completed=completed) + return + if row.event_type == "interaction.completed": + method, reason = payload.get(ACP_METHOD_KEY), payload.get(END_REASON_KEY) + if (row.lifecycle_state or None) in (None, "cancelled") and method in (None, CANCEL_METHOD) and not reason: + turn.block("cancel", row.occurred_at) + else: + turn.block("lifecycle", row.occurred_at, acp_method=method, end_reason=reason) + return + if row.event_type == "interaction.started": + turn.block("lifecycle", row.occurred_at, acp_method=payload.get(ACP_METHOD_KEY), end_reason=None) + return + if row.event_type in _ERROR_EVENTS: + message, _ = content_text(payload.get("error_message")) + turn.block("error", row.occurred_at, error_code=payload.get("error_code"), message=_cap(message)[0]) + return + if row.event_type in ("usage.updated", "unknown_source_event"): + # Unmapped protocol messages carry nothing a reader could use. + return + turn.block("event", row.occurred_at, event_type=row.event_type) + + +def build_page( + rows: Iterable[TraceRow], + *, + limit: int, + start_index: int = 0, + max_events: int = MAX_PAGE_EVENTS, +) -> tuple[list[dict[str, Any]], Optional[TraceRow], int]: + """Up to ``limit`` turns from ``rows``. + + Returns the serialized turns, the first row of the next page (``None`` at + the end) and the turn index the next page starts from. ``start_index`` is + the number of turns before this page, so turn numbers continue. + """ + turns: list[_Turn] = [] + current: Optional[_Turn] = None + index = start_index + events = 0 + prompts_on_page = 0 + for row in rows: + starts_turn = _is_turn_start(row) + if starts_turn and prompts_on_page >= limit: + return [turn.serialize() for turn in turns], row, index + if events >= max_events: + if current is not None: + current.continues = True + return [turn.serialize() for turn in turns], row, index + events += 1 + if starts_turn: + index += 1 + prompts_on_page += 1 + current = _Turn( + index=index, + kind="turn", + started_at=row.occurred_at, + turn_id=row.correlations.get("turn_id"), + research_session_id=row.research_session_id, + ) + current.block("prompt", row.occurred_at, text=None, redacted=False, truncated=False, captured=False) + turns.append(current) + continue + if current is None: + # Before the page's first prompt: the chat opening, or the rest of + # a turn that began on the previous page. + current = _Turn( + index=index, + kind="preamble" if index == 0 else "continued", + started_at=row.occurred_at, + research_session_id=row.research_session_id, + ) + turns.append(current) + _apply(current, row) + return [turn.serialize() for turn in turns], None, index diff --git a/src/research/analysis/study_traces/store.py b/src/research/analysis/study_traces/store.py new file mode 100644 index 0000000..59eb321 --- /dev/null +++ b/src/research/analysis/study_traces/store.py @@ -0,0 +1,147 @@ +"""Read one ACP chat's retained events in a stable, resumable order.""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Optional + +from sqlalchemy import String, and_, cast, func, literal, or_, select, tuple_ + +from database.research_schemas import ResearchEvent + +if TYPE_CHECKING: + from collections.abc import Iterator + from datetime import datetime + + from sqlalchemy.orm import Session + +__all__ = ["TraceRow", "iter_chat_events"] + +_ENVELOPE = ResearchEvent.envelope_json +_DELETED = "DELETED" +_ACP_SOURCE = "acp" +_RELAY_SOURCE = "relay" +_BATCH_SIZE = 500 +#: The tool call an event belongs to (correlations first, as the proxy writes it). +_TOOL_CALL_ID = func.coalesce( + _ENVELOPE["correlations"]["tool_call_id"].astext, _ENVELOPE["payload"]["tool_call_id"].astext +) + +#: Keyset order: time, then the emitter's own sequence, then the event id. +_ORDER = ( + ResearchEvent.occurred_at, + ResearchEvent.emitter_id, + ResearchEvent.emitter_sequence, + ResearchEvent.event_id, +) + + +@dataclass(frozen=True) +class TraceRow: + """One retained ACP event of a chat, with its stored envelope parts.""" + + event_id: str + occurred_at: datetime + emitter_id: str + emitter_sequence: int + research_session_id: Optional[str] + event_type: str + source: str + lifecycle_state: Optional[str] = None + payload: dict[str, Any] = field(default_factory=dict) + correlations: dict[str, Any] = field(default_factory=dict) + metrics: dict[str, Any] = field(default_factory=dict) + + @property + def key(self) -> tuple[datetime, str, int, str]: + return (self.occurred_at, self.emitter_id, self.emitter_sequence, self.event_id) + + +def _as_dict(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def iter_chat_events( + session: Session, + study_id: uuid.UUID, + enrollment_id: uuid.UUID, + chat_id: str, + *, + start_at: Optional[tuple[datetime, str, int, str]] = None, +) -> Iterator[TraceRow]: + """The chat's events in keyset order, from ``start_at`` (inclusive) on. + + Scoped to the study and enrollment; retention tombstones never appear. A + chat is the ACP session id the participant proxy stamps on its events. The + built-in agent's own report of a "Revise…" form (a decision carrying + ``elicitation_action``) has no chat id, so it joins through its tool call. + """ + scope = ( + ResearchEvent.study_id == study_id, + ResearchEvent.enrollment_id == enrollment_id, + ResearchEvent.retention_state != _DELETED, + ) + in_chat = and_( + ResearchEvent.source == _ACP_SOURCE, + _ENVELOPE["payload"]["session_id"].astext == chat_id, + ) + chat_tool_calls = select(_TOOL_CALL_ID).where(*scope, in_chat, _TOOL_CALL_ID.is_not(None)) + revision_reports = and_( + ResearchEvent.source == _RELAY_SOURCE, + ResearchEvent.event_type == "permission.decided", + _ENVELOPE["payload"].has_key("elicitation_action"), + _TOOL_CALL_ID.in_(chat_tool_calls), + ) + statement = select( + cast(ResearchEvent.event_id, String), + ResearchEvent.occurred_at, + ResearchEvent.emitter_id, + ResearchEvent.emitter_sequence, + cast(ResearchEvent.research_session_id, String), + ResearchEvent.event_type, + ResearchEvent.source, + _ENVELOPE["lifecycle_state"].astext, + _ENVELOPE["payload"], + _ENVELOPE["correlations"], + _ENVELOPE["metrics"], + ).where(*scope, or_(in_chat, revision_reports)) + if start_at is not None: + occurred_at, emitter_id, emitter_sequence, event_id = start_at + statement = statement.where( + tuple_(*_ORDER) + >= tuple_( + literal(occurred_at), + literal(emitter_id), + literal(int(emitter_sequence)), + literal(uuid.UUID(str(event_id))), + ) + ) + statement = statement.order_by(*(column.asc() for column in _ORDER)) + for row in session.execute(statement.execution_options(yield_per=_BATCH_SIZE)): + ( + event_id, + occurred_at, + emitter_id, + emitter_sequence, + research_session_id, + event_type, + source, + lifecycle_state, + payload, + correlations, + metrics, + ) = tuple(row) + yield TraceRow( + event_id=event_id, + occurred_at=occurred_at, + emitter_id=emitter_id or "", + emitter_sequence=int(emitter_sequence or 0), + research_session_id=research_session_id, + event_type=event_type, + source=source, + lifecycle_state=lifecycle_state, + payload=_as_dict(payload), + correlations=_as_dict(correlations), + metrics=_as_dict(metrics), + ) diff --git a/src/research/budget/__init__.py b/src/research/budget/__init__.py new file mode 100644 index 0000000..beabf12 --- /dev/null +++ b/src/research/budget/__init__.py @@ -0,0 +1,73 @@ +"""Participant inference budgets for the shared provider key. + +See ``ledger`` for the accounting invariant, ``meter`` for the per-call wrapper +used by the relay endpoints, ``estimate``/``pricing``/``usage`` for the +arithmetic and ``capability`` for the bearer Goose presents to the gateway. +""" + +from .capability import ( + INFERENCE_AUDIENCE, + INFERENCE_SCOPE, + decode_capability_bearer, + encode_capability_bearer, + issue_inference_capability, + verify_inference_capability, +) +from .errors import ( + AdjustmentInvalid, + BalanceMissing, + BudgetError, + BudgetRefused, + IdempotencyConflict, + PriceMissing, +) +from .meter import ( + AVAILABLE_HEADER, + InferenceMeter, + format_usd, + price_missing_response, + quota_refusal_response, +) +from .models import ( + AdjustmentRecord, + BalanceView, + HoldPlan, + ModelPrice, + Reservation, + SettlementOutcome, + UsageSnapshot, +) +from .pricing import MICRO_USD_PER_USD, cost_micro_usd, get_model_price, usd_to_micro +from .settings import BudgetSettings + +__all__ = [ + "AVAILABLE_HEADER", + "AdjustmentInvalid", + "AdjustmentRecord", + "BalanceMissing", + "BalanceView", + "BudgetError", + "BudgetRefused", + "BudgetSettings", + "HoldPlan", + "INFERENCE_AUDIENCE", + "INFERENCE_SCOPE", + "IdempotencyConflict", + "InferenceMeter", + "MICRO_USD_PER_USD", + "ModelPrice", + "PriceMissing", + "Reservation", + "SettlementOutcome", + "UsageSnapshot", + "cost_micro_usd", + "decode_capability_bearer", + "encode_capability_bearer", + "format_usd", + "get_model_price", + "issue_inference_capability", + "price_missing_response", + "quota_refusal_response", + "usd_to_micro", + "verify_inference_capability", +] diff --git a/src/research/budget/capability.py b/src/research/budget/capability.py new file mode 100644 index 0000000..ec47b77 --- /dev/null +++ b/src/research/budget/capability.py @@ -0,0 +1,95 @@ +"""The inference capability Goose presents as its bearer token. + +It is the existing HMAC-signed :class:`SessionCapability` with the audience +``inference`` and the scope ``inference:relay``: minted at bootstrap, bound to +the enrollment/session/study, revoked through ``enrollment.revocation_epoch`` +and verified server-side on every call. Goose reads ``OPENAI_API_KEY`` once +per process, so the token is the base64url encoding of the capability's +canonical JSON (signature included); the server decodes it and re-verifies +the signature, so the encoding only has to round-trip the JSON content. +""" + +from __future__ import annotations + +import base64 +import binascii +import json +import re +import uuid +from datetime import datetime +from typing import Optional + +from research.canonical import canonical_bytes +from research.runtime.bootstrap.capability import ( + INFERENCE_AUDIENCE, + INFERENCE_SCOPE, + issue_inference_capability, + verify_capability, +) +from research.runtime.bootstrap.models import CapabilityVerification, SessionCapability + +__all__ = [ + "INFERENCE_AUDIENCE", + "INFERENCE_SCOPE", + "decode_capability_bearer", + "encode_capability_bearer", + "issue_inference_capability", + "verify_inference_capability", +] + +_BEARER_RE = re.compile(r"^[A-Za-z0-9_-]{16,8192}$") + + +def encode_capability_bearer(capability: SessionCapability) -> str: + """base64url (no padding) of the capability's canonical JSON, signature included.""" + payload = capability.model_dump(mode="json") + return base64.urlsafe_b64encode(canonical_bytes(payload)).decode("ascii").rstrip("=") + + +def decode_capability_bearer(token: Optional[str]) -> Optional[SessionCapability]: + """Parse a bearer back into a capability; ``None`` for anything malformed.""" + if not token: + return None + text = token.strip() + if text.lower().startswith("bearer "): + text = text[len("bearer ") :].strip() + if not _BEARER_RE.match(text): + return None + padded = text + "=" * (-len(text) % 4) + try: + raw = base64.urlsafe_b64decode(padded.encode("ascii")) + payload = json.loads(raw.decode("utf-8")) + except (binascii.Error, ValueError, UnicodeDecodeError): + return None + if not isinstance(payload, dict): + return None + try: + return SessionCapability.model_validate(payload) + except Exception: # noqa: BLE001 - any validation failure is "not a capability" + return None + + +def verify_inference_capability( + capability: SessionCapability, + *, + secret: str, + current_revocation_epoch: int, + expected_enrollment_id: uuid.UUID, + expected_study_id: uuid.UUID, + now: Optional[datetime] = None, +) -> CapabilityVerification: + """Signature, audience, scope, expiry, epoch and subject checks. + + The research session is deliberately not required to be live: budgets are + per enrollment, and an idle-ended session must not break a running chat. + """ + return verify_capability( + capability, + secret, + INFERENCE_AUDIENCE, + list(INFERENCE_SCOPE), + now, + current_revocation_epoch, + expected_enrollment_id=expected_enrollment_id, + expected_study_id=expected_study_id, + ) diff --git a/src/research/budget/errors.py b/src/research/budget/errors.py new file mode 100644 index 0000000..6654930 --- /dev/null +++ b/src/research/budget/errors.py @@ -0,0 +1,56 @@ +"""Typed failures of the budget ledger.""" + +from __future__ import annotations + +import uuid +from typing import Optional + +__all__ = [ + "AdjustmentInvalid", + "BalanceMissing", + "BudgetError", + "BudgetRefused", + "IdempotencyConflict", + "PriceMissing", +] + + +class BudgetError(Exception): + """Base class for ledger failures.""" + + +class PriceMissing(BudgetError): + """The frozen model has no price row on its connection (fail closed).""" + + def __init__(self, connection_id: Optional[uuid.UUID], model: str) -> None: + super().__init__(f"no price configured for model {model!r} on connection {connection_id}") + self.connection_id = connection_id + self.model = model + + +class BalanceMissing(BudgetError): + """No balance row exists for the enrollment (and it could not be created).""" + + def __init__(self, enrollment_id: uuid.UUID) -> None: + super().__init__(f"no inference balance for enrollment {enrollment_id}") + self.enrollment_id = enrollment_id + + +class BudgetRefused(BudgetError): + """The worst-case hold does not fit the available budget.""" + + def __init__(self, *, available_micro_usd: int, needed_micro_usd: int) -> None: + super().__init__( + f"budget refused: available {available_micro_usd} micro-USD, " + f"needed {needed_micro_usd} micro-USD" + ) + self.available_micro_usd = int(available_micro_usd) + self.needed_micro_usd = int(needed_micro_usd) + + +class IdempotencyConflict(BudgetError): + """An idempotency key was reused with a different request body.""" + + +class AdjustmentInvalid(BudgetError): + """An adjustment amount/kind is not acceptable.""" diff --git a/src/research/budget/estimate.py b/src/research/budget/estimate.py new file mode 100644 index 0000000..dc4edca --- /dev/null +++ b/src/research/budget/estimate.py @@ -0,0 +1,166 @@ +"""Worst-case hold planning and output capping for one Chat Completions call. + +The prompt estimate is tokenizer-free on purpose (no network tokenizer, no +model-specific vocabularies): UTF-8 bytes divided by a conservative +bytes-per-token constant plus per-message overhead, times a safety factor. +Over-estimating only makes a call near exhaustion fail a little early; the +settled charge always uses the provider's real token counts. +""" + +from __future__ import annotations + +import json +import math +from typing import Any, Optional + +from .errors import BudgetRefused +from .models import HoldPlan, ModelPrice +from .pricing import cost_micro_usd +from .settings import BudgetSettings + +__all__ = [ + "apply_output_cap", + "client_output_cap", + "estimate_prompt_tokens", + "minimal_call_micro_usd", + "plan_hold", + "upstream_kind", +] + + +def _utf8_bytes(value: Any) -> int: + if value is None: + return 0 + if isinstance(value, str): + return len(value.encode("utf-8", errors="replace")) + try: + return len(json.dumps(value, default=str, ensure_ascii=False).encode("utf-8")) + except (TypeError, ValueError): + return len(str(value).encode("utf-8", errors="replace")) + + +def estimate_prompt_tokens(openai_body: dict, settings: BudgetSettings) -> int: + """Conservative token estimate for the prompt side of ``openai_body``.""" + messages = openai_body.get("messages") or [] + if not isinstance(messages, list): + messages = [] + tools = openai_body.get("tools") or [] + message_bytes = sum(_utf8_bytes(message) for message in messages) + tool_bytes = _utf8_bytes(tools) if tools else 0 + raw = ( + message_bytes / settings.input_bytes_per_token + + len(messages) * settings.per_message_token_overhead + + tool_bytes / settings.input_bytes_per_token + + 8 + ) + return int(math.ceil(raw * settings.input_estimate_safety)) + + +def _positive_int(value: Any) -> Optional[int]: + if isinstance(value, bool): + return None + if isinstance(value, int) and value > 0: + return value + if isinstance(value, float) and value > 0 and value.is_integer(): + return int(value) + return None + + +def client_output_cap(openai_body: dict) -> Optional[int]: + """The smaller of the client's ``max_tokens`` / ``max_completion_tokens``, if any.""" + caps = [ + cap + for cap in ( + _positive_int(openai_body.get("max_tokens")), + _positive_int(openai_body.get("max_completion_tokens")), + ) + if cap is not None + ] + return min(caps) if caps else None + + +def minimal_call_micro_usd(price: ModelPrice, settings: BudgetSettings) -> int: + """What the cheapest sensible call costs: a tiny prompt plus the minimum output.""" + return cost_micro_usd(64, price.input_usd_per_million) + cost_micro_usd( + settings.min_output_tokens, price.output_usd_per_million + ) + + +def plan_hold( + price: ModelPrice, + *, + available_micro_usd: int, + estimated_prompt_tokens: int, + client_cap: Optional[int], + settings: BudgetSettings, +) -> HoldPlan: + """Choose the output cap the budget affords and the hold that bounds the call. + + Raises :class:`BudgetRefused` when even ``min_output_tokens`` of output + would not fit after the estimated input cost. + """ + input_cost = cost_micro_usd(estimated_prompt_tokens, price.input_usd_per_million) + remaining = int(available_micro_usd) - input_cost + output_price = price.output_usd_per_million + if output_price <= 0: + affordable = settings.output_token_ceiling + elif remaining <= 0: + affordable = 0 + else: + affordable = int(remaining // output_price) # floor(micro-USD / micro-USD-per-token) + cap = min(settings.output_token_ceiling, affordable) + # The smallest useful completion: the configured minimum, or less when the + # client itself asked for a shorter answer (a small max_tokens is a valid + # request, not a sign of an exhausted budget). + minimum = settings.min_output_tokens + if client_cap is not None: + cap = min(cap, client_cap) + minimum = min(minimum, client_cap) + if cap < minimum: + needed = input_cost + cost_micro_usd(minimum, output_price) + raise BudgetRefused(available_micro_usd=int(available_micro_usd), needed_micro_usd=needed) + return HoldPlan( + estimated_prompt_tokens=int(estimated_prompt_tokens), + input_cost_micro_usd=input_cost, + output_cap_tokens=int(cap), + output_cost_micro_usd=cost_micro_usd(int(cap), output_price), + ) + + +def upstream_kind(base_url: Optional[str]) -> str: + """``openai`` for api.openai.com, ``openrouter`` for openrouter.ai, else ``other``.""" + host = (base_url or "").lower() + if "api.openai.com" in host: + return "openai" + if "openrouter.ai" in host: + return "openrouter" + return "other" + + +def apply_output_cap(openai_body: dict, cap: int, *, upstream_base_url: Optional[str]) -> None: + """Bound the completion in place so the provider cannot exceed the hold. + + api.openai.com rejects ``max_tokens`` for reasoning models, so it gets + ``max_completion_tokens`` only; every other OpenAI-compatible host gets + ``max_tokens`` (and a client-sent ``max_completion_tokens`` is capped too). + OpenRouter additionally receives ``usage: {include: true}`` so the settled + charge can use its reported cost. Multiple completions are never allowed. + """ + cap = int(cap) + kind = upstream_kind(upstream_base_url) + if kind == "openai": + openai_body.pop("max_tokens", None) + openai_body["max_completion_tokens"] = cap + else: + openai_body["max_tokens"] = cap + if "max_completion_tokens" in openai_body: + existing = _positive_int(openai_body.get("max_completion_tokens")) + openai_body["max_completion_tokens"] = min(existing, cap) if existing else cap + if kind == "openrouter": + usage = openai_body.get("usage") + if not isinstance(usage, dict): + usage = {} + usage["include"] = True + openai_body["usage"] = usage + openai_body["n"] = 1 + openai_body.pop("best_of", None) diff --git a/src/research/budget/ledger.py b/src/research/budget/ledger.py new file mode 100644 index 0000000..0bed43f --- /dev/null +++ b/src/research/budget/ledger.py @@ -0,0 +1,1064 @@ +"""The budget ledger: reservations, settlement, adjustments and read models. + +Every function takes a SQLAlchemy ``Session`` and owns its transaction: it +commits on success and rolls back on failure, so callers hand in a fresh +session and close it afterwards. Amounts are integer micro-USD. + +The limit is enforced in exactly one place, the reservation statement + + UPDATE enrollment_inference_balance + SET reserved_micro_usd = reserved_micro_usd + :hold + WHERE enrollment_id = :e + AND limit_micro_usd - settled_micro_usd - reserved_micro_usd >= :hold + +executed after ``SELECT ... FOR UPDATE`` on the same row and committed together +with the ledger row. Concurrent reservers serialise on the row lock, the guard +is re-evaluated on the locked, current row, and a crash between the two +statements leaves neither. Holds are released only with proof that nothing was +generated (VOIDED); an unknown outcome charges the full hold (FORFEITED, or +EXPIRED when the next reservation finds a hold past its deadline). +""" + +from __future__ import annotations + +import hashlib +import logging +import uuid +from datetime import datetime, timedelta, timezone +from typing import Any, Optional, Sequence + +from sqlalchemy import text +from sqlalchemy.exc import IntegrityError + +from database.research_schemas import ( + ADJUSTMENT_APPLY_DEFAULT, + ADJUSTMENT_KINDS, + ADJUSTMENT_SET_LIMIT, + ADJUSTMENT_TOP_UP, + BUDGET_UNIT_MICRO_USD, + LIMIT_SOURCE_ADJUSTED, + LIMIT_SOURCE_BACKFILL, + LIMIT_SOURCE_STUDY_DEFAULT, + RESERVATION_EXPIRED, + RESERVATION_FORFEITED, + RESERVATION_RESERVED, + RESERVATION_SETTLED, + RESERVATION_VOIDED, +) +from research.canonical import canonical_json + +from .errors import AdjustmentInvalid, BalanceMissing, BudgetRefused, IdempotencyConflict +from .estimate import ( + apply_output_cap, + client_output_cap, + estimate_prompt_tokens, + minimal_call_micro_usd, + plan_hold, +) +from .models import ( + AdjustmentRecord, + BalanceView, + ModelPrice, + Reservation, + SettlementOutcome, + UsageSnapshot, +) +from .pricing import charge_for_usage +from .settings import BudgetSettings + +__all__ = [ + "adjust", + "balance_summary", + "balance_view", + "create_balance", + "ensure_balance", + "expire_stale_reservations", + "forfeit", + "list_adjustments", + "list_reservations", + "list_study_balances", + "participant_view", + "participant_views", + "reserve", + "settle", + "study_spend_summary", + "void", +] + +logger = logging.getLogger(__name__) + +_USAGE_SOURCE_FORFEIT = "hold_forfeit" + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +# --------------------------------------------------------------------------- +# Balance rows +# --------------------------------------------------------------------------- + + +def create_balance( + db, + *, + enrollment_id: uuid.UUID, + study_id: uuid.UUID, + limit_micro_usd: int, + limit_source: str = LIMIT_SOURCE_STUDY_DEFAULT, + now: Optional[datetime] = None, + commit: bool = False, +) -> None: + """Insert a balance row inside the caller's transaction (enrollment creation).""" + if int(limit_micro_usd) < 0: + raise AdjustmentInvalid("a budget limit cannot be negative") + timestamp = _now(now) + db.execute( + text( + """ + INSERT INTO public.enrollment_inference_balance + (enrollment_id, study_id, unit, limit_micro_usd, limit_source, created_at, updated_at) + VALUES (:enrollment_id, :study_id, :unit, :limit, :source, :now, :now) + ON CONFLICT (enrollment_id) DO NOTHING + """ + ), + { + "enrollment_id": enrollment_id, + "study_id": study_id, + "unit": BUDGET_UNIT_MICRO_USD, + "limit": int(limit_micro_usd), + "source": limit_source, + "now": timestamp, + }, + ) + if commit: + db.commit() + + +def ensure_balance(db, enrollment_id: uuid.UUID, *, now: Optional[datetime] = None) -> bool: + """Create the balance row from the study default when missing; return whether it exists.""" + timestamp = _now(now) + try: + db.execute( + text( + """ + INSERT INTO public.enrollment_inference_balance + (enrollment_id, study_id, unit, limit_micro_usd, limit_source, created_at, updated_at) + SELECT e.enrollment_id, e.study_id, :unit, s.inference_budget_default_micro_usd, + :source, :now, :now + FROM public.research_enrollment e + JOIN public.study s ON s.study_id = e.study_id + WHERE e.enrollment_id = :enrollment_id + ON CONFLICT (enrollment_id) DO NOTHING + """ + ), + { + "enrollment_id": enrollment_id, + "unit": BUDGET_UNIT_MICRO_USD, + "source": LIMIT_SOURCE_BACKFILL, + "now": timestamp, + }, + ) + db.commit() + except Exception: + db.rollback() + raise + exists = db.execute( + text( + "SELECT 1 FROM public.enrollment_inference_balance WHERE enrollment_id = :enrollment_id" + ), + {"enrollment_id": enrollment_id}, + ).scalar_one_or_none() + return exists is not None + + +def _lock_balance(db, enrollment_id: uuid.UUID) -> Optional[dict]: + row = ( + db.execute( + text( + """ + SELECT enrollment_id, study_id, limit_micro_usd, settled_micro_usd, + reserved_micro_usd, limit_source, exhausted_at + FROM public.enrollment_inference_balance + WHERE enrollment_id = :enrollment_id + FOR UPDATE + """ + ), + {"enrollment_id": enrollment_id}, + ) + .mappings() + .first() + ) + return dict(row) if row is not None else None + + +def _expire_locked(db, enrollment_id: uuid.UUID, now: datetime) -> int: + """Charge every hold past its deadline in full. Caller holds the row lock.""" + total = db.execute( + text( + """ + WITH expired AS ( + UPDATE public.inference_reservation + SET state = :expired, + charged_micro_usd = hold_micro_usd, + usage_source = :source, + resolution_reason = 'expired', + resolved_at = :now + WHERE enrollment_id = :enrollment_id + AND state = :reserved + AND deadline_at < :now + RETURNING hold_micro_usd + ) + SELECT coalesce(sum(hold_micro_usd), 0) FROM expired + """ + ), + { + "enrollment_id": enrollment_id, + "expired": RESERVATION_EXPIRED, + "reserved": RESERVATION_RESERVED, + "source": _USAGE_SOURCE_FORFEIT, + "now": now, + }, + ).scalar_one() + total = int(total or 0) + if total > 0: + db.execute( + text( + """ + UPDATE public.enrollment_inference_balance + SET settled_micro_usd = settled_micro_usd + :total, + reserved_micro_usd = reserved_micro_usd - :total, + updated_at = :now + WHERE enrollment_id = :enrollment_id + """ + ), + {"enrollment_id": enrollment_id, "total": total, "now": now}, + ) + logger.warning( + "[Budget] expired %d micro-USD of stale holds for enrollment %s", + total, + enrollment_id, + ) + return total + + +def expire_stale_reservations( + db, enrollment_id: uuid.UUID, *, now: Optional[datetime] = None +) -> int: + """Standalone lazy expiry (also run inside every reservation).""" + timestamp = _now(now) + try: + if _lock_balance(db, enrollment_id) is None: + db.rollback() + return 0 + total = _expire_locked(db, enrollment_id, timestamp) + db.commit() + return total + except Exception: + db.rollback() + raise + + +# --------------------------------------------------------------------------- +# Reservation +# --------------------------------------------------------------------------- + + +def reserve( + db, + *, + enrollment_id: uuid.UUID, + study_id: uuid.UUID, + connection_id: Optional[uuid.UUID], + model: str, + entry_point: str, + request_id: str, + openai_body: dict, + upstream_base_url: Optional[str], + price: ModelPrice, + settings: BudgetSettings, + research_session_id: Optional[uuid.UUID] = None, + now: Optional[datetime] = None, +) -> Reservation: + """Hold the worst-case cost of ``openai_body`` or raise :class:`BudgetRefused`. + + On success the body has been capped in place (``apply_output_cap``) and a + ``RESERVED`` ledger row exists. On refusal nothing is reserved (only + ``refused_count`` is bumped), so a client retry of a 402 costs nothing. + """ + timestamp = _now(now) + estimated = estimate_prompt_tokens(openai_body, settings) + client_cap = client_output_cap(openai_body) + try: + row = _lock_balance(db, enrollment_id) + if row is None: + raise BalanceMissing(enrollment_id) + expired = _expire_locked(db, enrollment_id, timestamp) + limit = int(row["limit_micro_usd"]) + settled = int(row["settled_micro_usd"]) + expired + reserved = int(row["reserved_micro_usd"]) - expired + available = limit - settled - reserved + plan = plan_hold( + price, + available_micro_usd=available, + estimated_prompt_tokens=estimated, + client_cap=client_cap, + settings=settings, + ) + # ``call_count`` counts admitted calls (a later VOID keeps the count: + # the call was sent); refusals are counted separately. + available_after = db.execute( + text( + """ + UPDATE public.enrollment_inference_balance + SET reserved_micro_usd = reserved_micro_usd + :hold, + call_count = call_count + 1, + last_call_at = :now, + exhausted_at = NULL, + updated_at = :now + WHERE enrollment_id = :enrollment_id + AND limit_micro_usd - settled_micro_usd - reserved_micro_usd >= :hold + RETURNING limit_micro_usd - settled_micro_usd - reserved_micro_usd + """ + ), + {"enrollment_id": enrollment_id, "hold": plan.hold_micro_usd, "now": timestamp}, + ).scalar_one_or_none() + if available_after is None: + raise BudgetRefused(available_micro_usd=available, needed_micro_usd=plan.hold_micro_usd) + reservation_id = uuid.uuid4() + deadline = timestamp + timedelta(seconds=settings.reservation_deadline_seconds) + db.execute( + text( + """ + INSERT INTO public.inference_reservation + (reservation_id, enrollment_id, study_id, connection_id, model, entry_point, + request_id, research_session_id, state, hold_micro_usd, + estimated_prompt_tokens, output_cap_tokens, reserved_at, deadline_at) + VALUES (:reservation_id, :enrollment_id, :study_id, :connection_id, :model, + :entry_point, :request_id, :research_session_id, :state, :hold, + :estimated, :cap, :now, :deadline) + """ + ), + { + "reservation_id": reservation_id, + "enrollment_id": enrollment_id, + "study_id": study_id, + "connection_id": connection_id, + "model": model, + "entry_point": entry_point, + "request_id": request_id, + "research_session_id": research_session_id, + "state": RESERVATION_RESERVED, + "hold": plan.hold_micro_usd, + "estimated": plan.estimated_prompt_tokens, + "cap": plan.output_cap_tokens, + "now": timestamp, + "deadline": deadline, + }, + ) + db.commit() + except BudgetRefused as refused: + db.rollback() + _record_refusal(db, enrollment_id, refused, price, settings, timestamp) + raise + except Exception: + db.rollback() + raise + apply_output_cap(openai_body, plan.output_cap_tokens, upstream_base_url=upstream_base_url) + return Reservation( + reservation_id=reservation_id, + enrollment_id=enrollment_id, + study_id=study_id, + hold_micro_usd=plan.hold_micro_usd, + estimated_prompt_tokens=plan.estimated_prompt_tokens, + output_cap_tokens=plan.output_cap_tokens, + available_after_micro_usd=int(available_after), + deadline_at=deadline, + ) + + +def _record_refusal( + db, + enrollment_id: uuid.UUID, + refused: BudgetRefused, + price: ModelPrice, + settings: BudgetSettings, + now: datetime, +) -> None: + """Count the refusal; mark exhausted when not even a minimal call fits.""" + exhausted = refused.available_micro_usd < minimal_call_micro_usd(price, settings) + try: + db.execute( + text( + """ + UPDATE public.enrollment_inference_balance + SET refused_count = refused_count + 1, + exhausted_at = CASE WHEN :exhausted THEN coalesce(exhausted_at, :now) + ELSE exhausted_at END, + updated_at = :now + WHERE enrollment_id = :enrollment_id + """ + ), + {"enrollment_id": enrollment_id, "exhausted": exhausted, "now": now}, + ) + db.commit() + except Exception: # noqa: BLE001 - bookkeeping only; the refusal already stands + db.rollback() + logger.exception("[Budget] could not record refusal for enrollment %s", enrollment_id) + + +# --------------------------------------------------------------------------- +# Resolution: settle / forfeit / void +# --------------------------------------------------------------------------- + + +def _resolve( + db, + reservation_id: uuid.UUID, + *, + state: str, + charged_micro_usd: Optional[int], + charge_full_hold: bool, + prompt_tokens: Optional[int], + completion_tokens: Optional[int], + cached_prompt_tokens: Optional[int], + usage_source: Optional[str], + resolution_reason: Optional[str], + upstream_status: Optional[int], + finish_reason: Optional[str], + now: datetime, +) -> SettlementOutcome: + charged_sql = "hold_micro_usd" if charge_full_hold else ":charged" + try: + # Same lock order as ``reserve`` (balance row first, then the + # reservation rows), so a settlement racing a lazy expiry of the same + # hold serialises instead of deadlocking. + owner = db.execute( + text( + "SELECT enrollment_id FROM public.inference_reservation WHERE reservation_id = :reservation_id" + ), + {"reservation_id": reservation_id}, + ).scalar_one_or_none() + if owner is None: + db.rollback() + logger.warning("[Budget] reservation %s does not exist; %s ignored", reservation_id, state) + return SettlementOutcome( + reservation_id=reservation_id, state=state, charged_micro_usd=0, applied=False + ) + _lock_balance(db, owner) + row = ( + db.execute( + text( + f""" + UPDATE public.inference_reservation + SET state = :state, + charged_micro_usd = {charged_sql}, + prompt_tokens = :prompt_tokens, + completion_tokens = :completion_tokens, + cached_prompt_tokens = :cached_prompt_tokens, + usage_source = :usage_source, + resolution_reason = :resolution_reason, + upstream_status = :upstream_status, + finish_reason = :finish_reason, + resolved_at = :now + WHERE reservation_id = :reservation_id + AND state = :reserved + RETURNING enrollment_id, hold_micro_usd, charged_micro_usd + """ + ), + { + "reservation_id": reservation_id, + "state": state, + "charged": int(charged_micro_usd or 0), + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "cached_prompt_tokens": cached_prompt_tokens, + "usage_source": usage_source, + "resolution_reason": resolution_reason, + "upstream_status": upstream_status, + "finish_reason": finish_reason, + "now": now, + "reserved": RESERVATION_RESERVED, + }, + ) + .mappings() + .first() + ) + if row is None: + db.commit() + logger.warning( + "[Budget] reservation %s already resolved; %s ignored", reservation_id, state + ) + return SettlementOutcome( + reservation_id=reservation_id, state=state, charged_micro_usd=0, applied=False + ) + hold = int(row["hold_micro_usd"]) + charged = int(row["charged_micro_usd"] or 0) + db.execute( + text( + """ + UPDATE public.enrollment_inference_balance + SET reserved_micro_usd = reserved_micro_usd - :hold, + settled_micro_usd = settled_micro_usd + :charged, + settled_prompt_tokens = settled_prompt_tokens + :prompt_tokens, + settled_completion_tokens = settled_completion_tokens + :completion_tokens, + exhausted_at = CASE + WHEN limit_micro_usd - (settled_micro_usd + :charged) + - (reserved_micro_usd - :hold) <= 0 + THEN coalesce(exhausted_at, :now) + ELSE NULL END, + updated_at = :now + WHERE enrollment_id = :enrollment_id + """ + ), + { + "enrollment_id": row["enrollment_id"], + "hold": hold, + "charged": charged, + "prompt_tokens": int(prompt_tokens or 0), + "completion_tokens": int(completion_tokens or 0), + "now": now, + }, + ) + db.commit() + except Exception: + db.rollback() + raise + return SettlementOutcome( + reservation_id=reservation_id, state=state, charged_micro_usd=charged, applied=True + ) + + +def settle( + db, + reservation_id: uuid.UUID, + *, + usage: UsageSnapshot, + price: ModelPrice, + upstream_status: Optional[int] = None, + now: Optional[datetime] = None, +) -> SettlementOutcome: + """Replace the hold by the actual charge (which may exceed the hold).""" + charged, source = charge_for_usage(price, usage) + return _resolve( + db, + reservation_id, + state=RESERVATION_SETTLED, + charged_micro_usd=charged, + charge_full_hold=False, + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + cached_prompt_tokens=usage.cached_prompt_tokens, + usage_source=source, + resolution_reason="settled", + upstream_status=upstream_status, + finish_reason=usage.finish_reason, + now=_now(now), + ) + + +def forfeit( + db, + reservation_id: uuid.UUID, + *, + reason: str, + upstream_status: Optional[int] = None, + now: Optional[datetime] = None, +) -> SettlementOutcome: + """Outcome unknown: charge the full hold.""" + return _resolve( + db, + reservation_id, + state=RESERVATION_FORFEITED, + charged_micro_usd=None, + charge_full_hold=True, + prompt_tokens=None, + completion_tokens=None, + cached_prompt_tokens=None, + usage_source=_USAGE_SOURCE_FORFEIT, + resolution_reason=reason, + upstream_status=upstream_status, + finish_reason=None, + now=_now(now), + ) + + +def void( + db, + reservation_id: uuid.UUID, + *, + reason: str, + upstream_status: Optional[int] = None, + now: Optional[datetime] = None, +) -> SettlementOutcome: + """Proof that nothing was generated: release the hold.""" + return _resolve( + db, + reservation_id, + state=RESERVATION_VOIDED, + charged_micro_usd=0, + charge_full_hold=False, + prompt_tokens=None, + completion_tokens=None, + cached_prompt_tokens=None, + usage_source=None, + resolution_reason=reason, + upstream_status=upstream_status, + finish_reason=None, + now=_now(now), + ) + + +# --------------------------------------------------------------------------- +# Adjustments +# --------------------------------------------------------------------------- + + +def _adjustment_from_row(row: Any, *, replayed: bool = False) -> AdjustmentRecord: + return AdjustmentRecord( + adjustment_id=row["adjustment_id"], + enrollment_id=row["enrollment_id"], + study_id=row["study_id"], + kind=row["kind"], + delta_micro_usd=int(row["delta_micro_usd"]), + limit_before_micro_usd=int(row["limit_before_micro_usd"]), + limit_after_micro_usd=int(row["limit_after_micro_usd"]), + in_flight_micro_usd=int(row["in_flight_micro_usd"]), + reason=row["reason"], + actor=row["actor"], + idempotency_key=row["idempotency_key"], + occurred_at=row["occurred_at"], + replayed=replayed, + ) + + +def _find_adjustment(db, study_id: uuid.UUID, idempotency_key: str) -> Optional[dict]: + row = ( + db.execute( + text( + """ + SELECT * FROM public.inference_budget_adjustment + WHERE study_id = :study_id AND idempotency_key = :key + """ + ), + {"study_id": study_id, "key": idempotency_key}, + ) + .mappings() + .first() + ) + return dict(row) if row is not None else None + + +def _request_digest(enrollment_id: uuid.UUID, kind: str, amount: int, reason: Optional[str]) -> str: + payload = canonical_json( + {"enrollment_id": str(enrollment_id), "kind": kind, "amount": int(amount), "reason": reason} + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def adjust( + db, + *, + enrollment_id: uuid.UUID, + kind: str, + amount_micro_usd: int, + actor: Optional[str], + reason: Optional[str], + idempotency_key: Optional[str] = None, + now: Optional[datetime] = None, +) -> Optional[AdjustmentRecord]: + """Top up, set or apply-default the limit of one enrollment. + + ``TOP_UP`` adds ``amount`` (> 0); ``SET_LIMIT`` sets the limit to ``amount`` + (>= 0; lowering below committed spend is allowed and simply stops further + calls); ``APPLY_DEFAULT`` sets the limit to ``amount`` only for rows still + on the study default — ``STUDY_DEFAULT`` and ``BACKFILL`` rows, which were + filled from the default too — (manually adjusted rows keep theirs) and + returns ``None`` when nothing changed. A reused idempotency key replays the + original adjustment (``replayed=True``) or raises + :class:`IdempotencyConflict` when the body differs. + """ + if kind not in (ADJUSTMENT_TOP_UP, ADJUSTMENT_SET_LIMIT, ADJUSTMENT_APPLY_DEFAULT): + raise AdjustmentInvalid(f"unsupported adjustment kind {kind!r}") + amount = int(amount_micro_usd) + if kind == ADJUSTMENT_TOP_UP and amount <= 0: + raise AdjustmentInvalid("a top-up must be positive") + if kind != ADJUSTMENT_TOP_UP and amount < 0: + raise AdjustmentInvalid("a budget limit cannot be negative") + timestamp = _now(now) + digest = _request_digest(enrollment_id, kind, amount, reason) + try: + row = _lock_balance(db, enrollment_id) + if row is None: + raise BalanceMissing(enrollment_id) + study_id = row["study_id"] + if idempotency_key: + existing = _find_adjustment(db, study_id, idempotency_key) + if existing is not None: + db.rollback() + if existing.get("request_digest") != digest: + raise IdempotencyConflict( + f"idempotency key {idempotency_key!r} was used with a different request" + ) + return _adjustment_from_row(existing, replayed=True) + before = int(row["limit_micro_usd"]) + if kind == ADJUSTMENT_TOP_UP: + after = before + amount + source = LIMIT_SOURCE_ADJUSTED + elif kind == ADJUSTMENT_SET_LIMIT: + after = amount + source = LIMIT_SOURCE_ADJUSTED + else: + if row["limit_source"] == LIMIT_SOURCE_ADJUSTED or before == amount: + db.rollback() + return None + after = amount + source = LIMIT_SOURCE_STUDY_DEFAULT + in_flight = int(row["reserved_micro_usd"]) + db.execute( + text( + """ + UPDATE public.enrollment_inference_balance + SET limit_micro_usd = :after, + limit_source = :source, + exhausted_at = CASE + WHEN :after - settled_micro_usd - reserved_micro_usd > 0 THEN NULL + ELSE coalesce(exhausted_at, :now) END, + updated_at = :now + WHERE enrollment_id = :enrollment_id + """ + ), + {"enrollment_id": enrollment_id, "after": after, "source": source, "now": timestamp}, + ) + adjustment_id = uuid.uuid4() + db.execute( + text( + """ + INSERT INTO public.inference_budget_adjustment + (adjustment_id, enrollment_id, study_id, kind, delta_micro_usd, + limit_before_micro_usd, limit_after_micro_usd, in_flight_micro_usd, + reason, actor, idempotency_key, request_digest, occurred_at) + VALUES (:adjustment_id, :enrollment_id, :study_id, :kind, :delta, :before, :after, + :in_flight, :reason, :actor, :key, :digest, :now) + """ + ), + { + "adjustment_id": adjustment_id, + "enrollment_id": enrollment_id, + "study_id": study_id, + "kind": kind, + "delta": after - before, + "before": before, + "after": after, + "in_flight": in_flight, + "reason": reason, + "actor": actor, + "key": idempotency_key, + "digest": digest, + "now": timestamp, + }, + ) + db.commit() + except IntegrityError: + # Two identical requests raced on the same idempotency key: replay. + db.rollback() + if idempotency_key: + existing = _find_adjustment(db, study_id, idempotency_key) + if existing is not None and existing.get("request_digest") == digest: + return _adjustment_from_row(existing, replayed=True) + raise + except Exception: + db.rollback() + raise + return AdjustmentRecord( + adjustment_id=adjustment_id, + enrollment_id=enrollment_id, + study_id=study_id, + kind=kind, + delta_micro_usd=after - before, + limit_before_micro_usd=before, + limit_after_micro_usd=after, + in_flight_micro_usd=in_flight, + reason=reason, + actor=actor, + idempotency_key=idempotency_key, + occurred_at=timestamp, + ) + + +# --------------------------------------------------------------------------- +# Read models +# --------------------------------------------------------------------------- + +_BALANCE_COLUMNS = """ + enrollment_id, study_id, unit, limit_micro_usd, settled_micro_usd, reserved_micro_usd, + settled_prompt_tokens, settled_completion_tokens, call_count, refused_count, last_call_at, + limit_source, exhausted_at, updated_at +""" + + +def _balance_from_row(row: Any) -> BalanceView: + return BalanceView( + enrollment_id=row["enrollment_id"], + study_id=row["study_id"], + unit=row["unit"], + limit_micro_usd=int(row["limit_micro_usd"]), + settled_micro_usd=int(row["settled_micro_usd"]), + reserved_micro_usd=int(row["reserved_micro_usd"]), + settled_prompt_tokens=int(row["settled_prompt_tokens"]), + settled_completion_tokens=int(row["settled_completion_tokens"]), + call_count=int(row["call_count"]), + refused_count=int(row["refused_count"]), + last_call_at=row["last_call_at"], + limit_source=row["limit_source"], + exhausted_at=row["exhausted_at"], + updated_at=row["updated_at"], + ) + + +def balance_view(db, enrollment_id: uuid.UUID) -> Optional[BalanceView]: + row = ( + db.execute( + text( + f"SELECT {_BALANCE_COLUMNS} FROM public.enrollment_inference_balance " + "WHERE enrollment_id = :enrollment_id" + ), + {"enrollment_id": enrollment_id}, + ) + .mappings() + .first() + ) + return _balance_from_row(row) if row is not None else None + + +def balance_summary(view: BalanceView) -> dict: + """Researcher-facing balance numbers (micro-USD integers).""" + return { + "unit": view.unit, + "limit_micro_usd": view.limit_micro_usd, + "consumed_micro_usd": view.settled_micro_usd, + "reserved_micro_usd": view.reserved_micro_usd, + "remaining_micro_usd": max(view.available_micro_usd, 0), + "available_micro_usd": view.available_micro_usd, + "limit_source": view.limit_source, + "call_count": view.call_count, + "refused_count": view.refused_count, + "last_call_at": view.last_call_at.isoformat() if view.last_call_at else None, + "exhausted": view.exhausted, + "exhausted_at": view.exhausted_at.isoformat() if view.exhausted_at else None, + "updated_at": view.updated_at.isoformat() if view.updated_at else None, + } + + +def list_study_balances(db, study_id: uuid.UUID) -> list[BalanceView]: + rows = ( + db.execute( + text( + f"SELECT {_BALANCE_COLUMNS} FROM public.enrollment_inference_balance " + "WHERE study_id = :study_id ORDER BY updated_at DESC" + ), + {"study_id": study_id}, + ) + .mappings() + .all() + ) + return [_balance_from_row(row) for row in rows] + + +def participant_view( + db, *, enrollment_id: uuid.UUID, now: Optional[datetime] = None +) -> Optional[dict]: + """The arm-blind ``budget`` block shown to the participant and the plugin. + + Numbers only (unit, limit, consumed, reserved, remaining, fraction used, + warning/exhausted flags); never a model, price or profile name. + """ + row = ( + db.execute( + text( + """ + SELECT b.unit, b.limit_micro_usd, b.settled_micro_usd, b.reserved_micro_usd, + b.exhausted_at, b.updated_at, s.inference_budget_warning_fraction + FROM public.enrollment_inference_balance b + JOIN public.study s ON s.study_id = b.study_id + WHERE b.enrollment_id = :enrollment_id + """ + ), + {"enrollment_id": enrollment_id}, + ) + .mappings() + .first() + ) + if row is None: + return None + return _participant_block(row, _now(now)) + + +def participant_views( + db, enrollment_ids, *, now: Optional[datetime] = None +) -> dict[uuid.UUID, Optional[dict]]: + """``participant_view`` for several enrollments (one query).""" + ids = list(enrollment_ids) + if not ids: + return {} + rows = ( + db.execute( + text( + """ + SELECT b.enrollment_id, b.unit, b.limit_micro_usd, b.settled_micro_usd, + b.reserved_micro_usd, b.exhausted_at, b.updated_at, + s.inference_budget_warning_fraction + FROM public.enrollment_inference_balance b + JOIN public.study s ON s.study_id = b.study_id + WHERE b.enrollment_id = ANY(:ids) + """ + ), + {"ids": ids}, + ) + .mappings() + .all() + ) + timestamp = _now(now) + result: dict[uuid.UUID, Optional[dict]] = {enrollment_id: None for enrollment_id in ids} + for row in rows: + result[row["enrollment_id"]] = _participant_block(row, timestamp) + return result + + +def _participant_block(row: Any, now: datetime) -> dict: + limit = int(row["limit_micro_usd"]) + consumed = int(row["settled_micro_usd"]) + reserved = int(row["reserved_micro_usd"]) + available = limit - consumed - reserved + committed = consumed + reserved + fraction_used = 1.0 if limit <= 0 else min(1.0, committed / limit) + warning_fraction = float(row["inference_budget_warning_fraction"] or 0) + exhausted = row["exhausted_at"] is not None or available <= 0 + return { + "unit": row["unit"], + "limit": limit, + "consumed": consumed, + "reserved": reserved, + "remaining": max(available, 0), + "fraction_used": round(fraction_used, 4), + "warning_fraction": warning_fraction, + "warning": bool(exhausted or fraction_used >= warning_fraction), + "exhausted": bool(exhausted), + "exhausted_at": row["exhausted_at"].isoformat() if row["exhausted_at"] else None, + "as_of": now.isoformat(), + } + + +def study_spend_summary( + db, study_id: uuid.UUID, *, enrollment_ids: Optional[Sequence[uuid.UUID]] = None +) -> dict: + """Exact metered spend of a study from the ledger (settled + forfeited + expired). + + ``enrollment_ids`` restricts it to those participants (analytics filters). + """ + scope = "" + params: dict[str, Any] = {"study_id": study_id} + if enrollment_ids is not None: + scope = " AND enrollment_id = ANY(CAST(:enrollment_ids AS uuid[]))" + params["enrollment_ids"] = [str(item) for item in enrollment_ids] + row = ( + db.execute( + text( + """ + SELECT coalesce(sum(charged_micro_usd), 0) AS spend, + count(*) FILTER (WHERE state <> :voided) AS calls, + coalesce(sum(prompt_tokens), 0) AS prompt_tokens, + coalesce(sum(completion_tokens), 0) AS completion_tokens + FROM public.inference_reservation + WHERE study_id = :study_id + AND state IN (:settled, :forfeited, :expired) + """ + + scope + ), + { + **params, + "voided": RESERVATION_VOIDED, + "settled": RESERVATION_SETTLED, + "forfeited": RESERVATION_FORFEITED, + "expired": RESERVATION_EXPIRED, + }, + ) + .mappings() + .one() + ) + open_holds = db.execute( + text( + "SELECT coalesce(sum(reserved_micro_usd), 0) FROM public.enrollment_inference_balance " + "WHERE study_id = :study_id" + scope + ), + params, + ).scalar_one() + return { + "metered_spend_micro_usd": int(row["spend"]), + "metered_calls": int(row["calls"]), + "reserved_micro_usd": int(open_holds or 0), + "prompt_tokens": int(row["prompt_tokens"]), + "completion_tokens": int(row["completion_tokens"]), + } + + +def list_reservations( + db, + enrollment_id: uuid.UUID, + *, + limit: int = 50, + before: Optional[datetime] = None, +) -> list[dict]: + """Newest-first ledger page for one enrollment (``before`` = cursor).""" + rows = ( + db.execute( + text( + """ + SELECT reservation_id, model, entry_point, request_id, research_session_id, state, + hold_micro_usd, estimated_prompt_tokens, output_cap_tokens, charged_micro_usd, + prompt_tokens, completion_tokens, cached_prompt_tokens, usage_source, + resolution_reason, upstream_status, finish_reason, reserved_at, deadline_at, + resolved_at + FROM public.inference_reservation + WHERE enrollment_id = :enrollment_id + AND (CAST(:before AS timestamptz) IS NULL OR reserved_at < :before) + ORDER BY reserved_at DESC, reservation_id DESC + LIMIT :limit + """ + ), + {"enrollment_id": enrollment_id, "before": before, "limit": max(1, min(int(limit), 200))}, + ) + .mappings() + .all() + ) + return [dict(row) for row in rows] + + +def list_adjustments( + db, + enrollment_id: uuid.UUID, + *, + limit: int = 50, + before: Optional[datetime] = None, +) -> list[AdjustmentRecord]: + rows = ( + db.execute( + text( + """ + SELECT * FROM public.inference_budget_adjustment + WHERE enrollment_id = :enrollment_id + AND (CAST(:before AS timestamptz) IS NULL OR occurred_at < :before) + ORDER BY occurred_at DESC, adjustment_id DESC + LIMIT :limit + """ + ), + {"enrollment_id": enrollment_id, "before": before, "limit": max(1, min(int(limit), 200))}, + ) + .mappings() + .all() + ) + return [_adjustment_from_row(dict(row)) for row in rows] + + +# Exported for callers that need the vocabulary without importing the ORM module. +KINDS = ADJUSTMENT_KINDS diff --git a/src/research/budget/meter.py b/src/research/budget/meter.py new file mode 100644 index 0000000..aa0932e --- /dev/null +++ b/src/research/budget/meter.py @@ -0,0 +1,348 @@ +"""The meter the relay endpoints wrap around one upstream inference call. + +``reserve`` runs before the request is serialised (it caps the body in place); +exactly one ``resolve_*`` runs afterwards. All database work happens in the +threadpool on a fresh session (``App.get_db_session`` is a thread-local scoped +session, so the event-loop thread must never touch the ledger), and +resolutions are shielded from cancellation and never raise: the ledger's +``state = 'RESERVED'`` guard and the lazy expiry are the safety net. +""" + +from __future__ import annotations + +import json +import logging +import uuid +from decimal import Decimal +from typing import Any, Callable, Optional + +import anyio +import httpx +from fastapi import Response +from fastapi.concurrency import run_in_threadpool + +from . import ledger, pricing +from .errors import BalanceMissing, BudgetRefused, PriceMissing +from .models import ModelPrice, Reservation +from .settings import BudgetSettings +from .usage import extract_usage_from_json, extract_usage_from_sse + +__all__ = [ + "AVAILABLE_HEADER", + "InferenceMeter", + "budget_unavailable_response", + "budget_unconfigured_response", + "format_usd", + "price_missing_response", + "quota_refusal_response", +] + +logger = logging.getLogger(__name__) + +AVAILABLE_HEADER = "X-Inference-Budget-Available-Micro-USD" + +# Nothing was sent to the provider, so nothing can have been billed. +_VOID_TRANSPORT_ERRORS = ( + httpx.ConnectError, + httpx.ConnectTimeout, + httpx.PoolTimeout, + httpx.UnsupportedProtocol, + httpx.InvalidURL, + httpx.ProxyError, +) + + +def format_usd(micro_usd: int) -> str: + amount = Decimal(int(micro_usd)) / pricing.MICRO_USD_PER_USD + if amount != 0 and abs(amount) < Decimal("0.01"): + return f"${amount:.4f}" + return f"${amount:.2f}" + + +def quota_refusal_response(refused: BudgetRefused) -> Response: + """HTTP 402 in the OpenAI error shape (Goose shows the message; it does not retry).""" + available = max(refused.available_micro_usd, 0) + message = ( + "Your study's AI budget is used up " + f"(available {format_usd(available)}; this request needs at least " + f"{format_usd(refused.needed_micro_usd)}). Ask the study team for a top-up." + ) + body = { + "error": { + "message": message, + "type": "insufficient_quota", + "code": "quota_exhausted", + "available_micro_usd": available, + "needed_micro_usd": refused.needed_micro_usd, + } + } + return Response(content=json.dumps(body), status_code=402, media_type="application/json") + + +def price_missing_response() -> Response: + """HTTP 503: the arm's model has no price. Participant-facing: no model name.""" + body = { + "error": { + "message": ( + "The study's model has no budget price configured on its provider " + "connection, so metered inference is refused. Ask the study team." + ), + "type": "server_error", + "code": "price_missing", + } + } + return Response(content=json.dumps(body), status_code=503, media_type="application/json") + + +def budget_unconfigured_response() -> Response: + """HTTP 402: the enrollment has no balance row (fail closed, no amounts).""" + body = { + "error": { + "message": ( + "No AI budget has been set up for this study enrollment yet, so the " + "request was refused. Ask the study team." + ), + "type": "insufficient_quota", + "code": "quota_exhausted", + } + } + return Response(content=json.dumps(body), status_code=402, media_type="application/json") + + +def budget_unavailable_response() -> Response: + """HTTP 503: the ledger could not be consulted; refuse rather than spend unmetered.""" + body = { + "error": { + "message": "The study budget could not be checked right now. Try again shortly.", + "type": "server_error", + "code": "budget_unavailable", + } + } + return Response(content=json.dumps(body), status_code=503, media_type="application/json") + + +class InferenceMeter: + """Reserve → forward → resolve for one call on one enrollment.""" + + def __init__( + self, + *, + app: Any, + enrollment_id: uuid.UUID, + study_id: uuid.UUID, + connection_id: Optional[uuid.UUID], + model: str, + entry_point: str, + research_session_id: Optional[uuid.UUID] = None, + settings: Optional[BudgetSettings] = None, + price: Optional[ModelPrice] = None, + ) -> None: + self.app = app + self.enrollment_id = enrollment_id + self.study_id = study_id + self.connection_id = connection_id + self.model = model + self.entry_point = entry_point + self.research_session_id = research_session_id + self.settings = settings or BudgetSettings.from_env() + self._price = price + self._reservation: Optional[Reservation] = None + self._resolved = False + + # -- state ------------------------------------------------------------- + + @property + def reservation(self) -> Optional[Reservation]: + return self._reservation + + @property + def resolved(self) -> bool: + return self._resolved + + @property + def available_after_reserve(self) -> Optional[int]: + return None if self._reservation is None else self._reservation.available_after_micro_usd + + def response_headers(self) -> dict[str, str]: + available = self.available_after_reserve + return {} if available is None else {AVAILABLE_HEADER: str(available)} + + # -- reserve ----------------------------------------------------------- + + def _reserve_sync(self, openai_body: dict, request_id: str, upstream_base_url: Optional[str]): + db = self.app.get_db_session() + try: + if self._price is None: + self._price = pricing.get_model_price(db, self.connection_id, self.model) + if not ledger.ensure_balance(db, self.enrollment_id): + raise BalanceMissing(self.enrollment_id) + return ledger.reserve( + db, + enrollment_id=self.enrollment_id, + study_id=self.study_id, + connection_id=self.connection_id, + model=self.model, + entry_point=self.entry_point, + request_id=request_id, + openai_body=openai_body, + upstream_base_url=upstream_base_url, + price=self._price, + settings=self.settings, + research_session_id=self.research_session_id, + ) + finally: + db.close() + + async def reserve( + self, openai_body: dict, *, request_id: str, upstream_base_url: Optional[str] + ) -> Optional[Response]: + """Hold the worst-case cost; return a refusal response or ``None`` on success.""" + try: + self._reservation = await run_in_threadpool( + self._reserve_sync, openai_body, request_id, upstream_base_url + ) + except PriceMissing as exc: + # The model name stays in the server log; the participant sees no arm detail. + logger.error("[Budget] %s (entry_point=%s)", exc, self.entry_point) + return price_missing_response() + except BalanceMissing as exc: + logger.error("[Budget] %s (entry_point=%s)", exc, self.entry_point) + return budget_unconfigured_response() + except BudgetRefused as refused: + logger.info( + "[Budget] refused enrollment=%s available=%d needed=%d entry_point=%s", + self.enrollment_id, + refused.available_micro_usd, + refused.needed_micro_usd, + self.entry_point, + ) + return quota_refusal_response(refused) + except Exception: # noqa: BLE001 - never spend unmetered on a ledger failure + logger.exception( + "[Budget] reservation failed for enrollment=%s entry_point=%s; refusing", + self.enrollment_id, + self.entry_point, + ) + return budget_unavailable_response() + logger.info( + "[Budget] reserved %d micro-USD (cap=%d tokens, est_in=%d) enrollment=%s request=%s", + self._reservation.hold_micro_usd, + self._reservation.output_cap_tokens, + self._reservation.estimated_prompt_tokens, + self.enrollment_id, + request_id, + ) + return None + + # -- resolve ----------------------------------------------------------- + + async def _run_resolution(self, fn: Callable[[], Any], label: str) -> None: + if self._reservation is None or self._resolved: + return + self._resolved = True + with anyio.CancelScope(shield=True): + try: + await run_in_threadpool(fn) + except Exception: # noqa: BLE001 - never let bookkeeping break the response + logger.exception( + "[Budget] %s failed for reservation %s; lazy expiry will charge the hold", + label, + self._reservation.reservation_id, + ) + + def _with_session(self, fn: Callable[[Any], Any]) -> Callable[[], Any]: + def run(): + db = self.app.get_db_session() + try: + return fn(db) + finally: + db.close() + + return run + + async def resolve_usage(self, usage, *, upstream_status: Optional[int], missing_reason: str) -> None: + reservation = self._reservation + if reservation is None: + return + if usage is None or not usage.has_tokens: + await self._run_resolution( + self._with_session( + lambda db: ledger.forfeit( + db, reservation.reservation_id, reason=missing_reason, + upstream_status=upstream_status, + ) + ), + "forfeit", + ) + return + price = self._price + await self._run_resolution( + self._with_session( + lambda db: ledger.settle( + db, reservation.reservation_id, usage=usage, price=price, + upstream_status=upstream_status, + ) + ), + "settle", + ) + + async def resolve_stream(self, raw_sse: str, *, upstream_status: Optional[int]) -> None: + """Settle from the buffered stream; no usage chunk means the hold is forfeited.""" + await self.resolve_usage( + extract_usage_from_sse(raw_sse or ""), + upstream_status=upstream_status, + missing_reason="usage_missing", + ) + + async def resolve_response(self, body: Any, *, upstream_status: Optional[int]) -> None: + await self.resolve_usage( + extract_usage_from_json(body), upstream_status=upstream_status, + missing_reason="usage_missing", + ) + + async def resolve_transport_error(self, error: BaseException) -> None: + """Connect-phase failures are voided; anything after the request left is forfeited.""" + reservation = self._reservation + if reservation is None: + return + reason = f"transport:{type(error).__name__}" + if isinstance(error, _VOID_TRANSPORT_ERRORS): + await self._run_resolution( + self._with_session( + lambda db: ledger.void(db, reservation.reservation_id, reason=reason) + ), + "void", + ) + return + await self._run_resolution( + self._with_session( + lambda db: ledger.forfeit(db, reservation.reservation_id, reason=reason) + ), + "forfeit", + ) + + async def resolve_upstream_error(self, status: int) -> None: + """An upstream 4xx/5xx before generation: nothing billed, release the hold.""" + reservation = self._reservation + if reservation is None: + return + await self._run_resolution( + self._with_session( + lambda db: ledger.void( + db, reservation.reservation_id, reason="upstream_status", + upstream_status=int(status), + ) + ), + "void", + ) + + async def resolve_timeout(self) -> None: + reservation = self._reservation + if reservation is None: + return + await self._run_resolution( + self._with_session( + lambda db: ledger.forfeit(db, reservation.reservation_id, reason="stream_timeout") + ), + "forfeit", + ) diff --git a/src/research/budget/models.py b/src/research/budget/models.py new file mode 100644 index 0000000..873b877 --- /dev/null +++ b/src/research/budget/models.py @@ -0,0 +1,127 @@ +"""Plain value objects shared by the ledger, pricing and meter modules.""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from datetime import datetime +from decimal import Decimal +from typing import Optional + +__all__ = [ + "AdjustmentRecord", + "BalanceView", + "HoldPlan", + "ModelPrice", + "Reservation", + "SettlementOutcome", + "UsageSnapshot", +] + + +@dataclass(frozen=True) +class ModelPrice: + """USD per million tokens (== micro-USD per token) for one connection/model.""" + + connection_id: Optional[uuid.UUID] + model: str + input_usd_per_million: Decimal + output_usd_per_million: Decimal + cached_input_usd_per_million: Optional[Decimal] = None + + +@dataclass(frozen=True) +class UsageSnapshot: + """Token usage read from an upstream response (stream or JSON).""" + + prompt_tokens: Optional[int] + completion_tokens: Optional[int] + cached_prompt_tokens: Optional[int] = None + #: Provider-reported cost in USD (OpenRouter ``usage.cost``); exact when present. + cost_usd: Optional[Decimal] = None + finish_reason: Optional[str] = None + + @property + def has_tokens(self) -> bool: + return self.prompt_tokens is not None or self.completion_tokens is not None + + +@dataclass(frozen=True) +class HoldPlan: + """Worst-case cost of one call and the output cap that bounds it.""" + + estimated_prompt_tokens: int + input_cost_micro_usd: int + output_cap_tokens: int + output_cost_micro_usd: int + + @property + def hold_micro_usd(self) -> int: + return self.input_cost_micro_usd + self.output_cost_micro_usd + + +@dataclass(frozen=True) +class Reservation: + """A committed hold (the ledger row exists and the balance counts it).""" + + reservation_id: uuid.UUID + enrollment_id: uuid.UUID + study_id: uuid.UUID + hold_micro_usd: int + estimated_prompt_tokens: int + output_cap_tokens: int + available_after_micro_usd: int + deadline_at: datetime + + +@dataclass(frozen=True) +class SettlementOutcome: + """Result of settle/forfeit/void: ``applied`` is False for an idempotent no-op.""" + + reservation_id: uuid.UUID + state: str + charged_micro_usd: int + applied: bool + + +@dataclass(frozen=True) +class BalanceView: + enrollment_id: uuid.UUID + study_id: uuid.UUID + unit: str + limit_micro_usd: int + settled_micro_usd: int + reserved_micro_usd: int + settled_prompt_tokens: int + settled_completion_tokens: int + call_count: int + refused_count: int + last_call_at: Optional[datetime] + limit_source: str + exhausted_at: Optional[datetime] + updated_at: datetime + + @property + def available_micro_usd(self) -> int: + return self.limit_micro_usd - self.settled_micro_usd - self.reserved_micro_usd + + @property + def exhausted(self) -> bool: + return self.exhausted_at is not None or self.available_micro_usd <= 0 + + +@dataclass(frozen=True) +class AdjustmentRecord: + adjustment_id: uuid.UUID + enrollment_id: uuid.UUID + study_id: uuid.UUID + kind: str + delta_micro_usd: int + limit_before_micro_usd: int + limit_after_micro_usd: int + in_flight_micro_usd: int + reason: Optional[str] + actor: Optional[str] + idempotency_key: Optional[str] + occurred_at: datetime + replayed: bool = False diff --git a/src/research/budget/pricing.py b/src/research/budget/pricing.py new file mode 100644 index 0000000..4cd5237 --- /dev/null +++ b/src/research/budget/pricing.py @@ -0,0 +1,137 @@ +"""Prices and charges. USD per million tokens equals micro-USD per token.""" + +from __future__ import annotations + +import math +import uuid +from decimal import Decimal, InvalidOperation +from typing import Any, Optional + +from sqlalchemy import select + +from database.db_schemas import ProviderModelPrice + +from .errors import PriceMissing +from .models import ModelPrice, UsageSnapshot + +__all__ = [ + "MAX_USD_AMOUNT", + "MICRO_USD_PER_USD", + "charge_for_usage", + "cost_micro_usd", + "get_model_price", + "parse_usd_amount", + "parse_usd_per_million", + "price_from_row", + "usd_to_micro", +] + +MICRO_USD_PER_USD = Decimal(1_000_000) +#: Upper bound for any single budget amount or price (sanity, not policy). +MAX_USD_AMOUNT = Decimal("100000") + + +def _parse_decimal(value, *, what: str, maximum: Decimal) -> Decimal: + text = str(value).strip() if value is not None else "" + if not text: + raise ValueError(f"{what} is required") + try: + amount = Decimal(text) + except InvalidOperation as exc: + raise ValueError(f"{what} must be a decimal number") from exc + if not amount.is_finite(): + raise ValueError(f"{what} must be a finite number") + if amount < 0: + raise ValueError(f"{what} must not be negative") + if amount > maximum: + raise ValueError(f"{what} must not exceed {maximum}") + if amount.as_tuple().exponent < -6: + raise ValueError(f"{what} may have at most 6 decimal places") + return amount + + +def parse_usd_amount(value) -> Decimal: + """A USD budget amount from the API boundary (string in, Decimal out).""" + return _parse_decimal(value, what="amount", maximum=MAX_USD_AMOUNT) + + +def parse_usd_per_million(value) -> Decimal: + """A price in USD per million tokens from the API boundary.""" + return _parse_decimal(value, what="price", maximum=Decimal("10000")) + + +def usd_to_micro(value: Decimal | str | int | float) -> int: + """Convert a USD amount to integer micro-USD, rounding up.""" + try: + amount = Decimal(str(value)) + except InvalidOperation as exc: + raise ValueError(f"not a decimal amount: {value!r}") from exc + if not amount.is_finite(): + raise ValueError(f"not a finite amount: {value!r}") + return int(math.ceil(amount * MICRO_USD_PER_USD)) + + +def cost_micro_usd(tokens: Optional[int], usd_per_million: Decimal) -> int: + """``ceil(tokens * price)``; zero for missing/non-positive token counts.""" + if tokens is None or tokens <= 0: + return 0 + return int(math.ceil(Decimal(int(tokens)) * Decimal(usd_per_million))) + + +def price_from_row(row: Any) -> ModelPrice: + return ModelPrice( + connection_id=getattr(row, "connection_id", None), + model=str(row.model), + input_usd_per_million=Decimal(row.input_usd_per_million), + output_usd_per_million=Decimal(row.output_usd_per_million), + cached_input_usd_per_million=( + None + if row.cached_input_usd_per_million is None + else Decimal(row.cached_input_usd_per_million) + ), + ) + + +def get_model_price(db, connection_id: Optional[uuid.UUID], model: str) -> ModelPrice: + """Return the price row for ``(connection_id, model)`` or raise :class:`PriceMissing`. + + There is deliberately no default price: a metered model without a price + cannot be held or settled, so the call is refused before anything is sent. + """ + if connection_id is None or not model: + raise PriceMissing(connection_id, model) + row = db.execute( + select(ProviderModelPrice).where( + ProviderModelPrice.connection_id == connection_id, + ProviderModelPrice.model == model, + ) + ).scalars().first() + if row is None: + raise PriceMissing(connection_id, model) + return price_from_row(row) + + +def charge_for_usage(price: ModelPrice, usage: UsageSnapshot) -> tuple[int, str]: + """Return ``(charged_micro_usd, usage_source)`` for settled usage. + + A provider-reported cost (OpenRouter ``usage.cost``) is exact and wins. + Otherwise cached prompt tokens bill at the cached price (or the input price + when none is configured) and completion tokens (which already include + reasoning tokens on OpenAI/OpenRouter) bill at the output price. + """ + if usage.cost_usd is not None: + return usd_to_micro(usage.cost_usd), "provider_cost" + prompt = int(usage.prompt_tokens or 0) + cached = min(int(usage.cached_prompt_tokens or 0), prompt) + uncached = max(prompt - cached, 0) + cached_price = ( + price.cached_input_usd_per_million + if price.cached_input_usd_per_million is not None + else price.input_usd_per_million + ) + charged = ( + cost_micro_usd(uncached, price.input_usd_per_million) + + cost_micro_usd(cached, cached_price) + + cost_micro_usd(usage.completion_tokens, price.output_usd_per_million) + ) + return charged, "price_table" diff --git a/src/research/budget/settings.py b/src/research/budget/settings.py new file mode 100644 index 0000000..44cff03 --- /dev/null +++ b/src/research/budget/settings.py @@ -0,0 +1,93 @@ +"""Tunables for the metered inference relay (read from the environment). + +Every knob has a conservative default so a deployment without any of these +variables still enforces budgets. Values are read at call time (like the other +research knobs) so a running backend picks up changes on restart only. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Mapping, Optional + +__all__ = ["BudgetSettings"] + + +def _int(environ: Mapping[str, str], name: str, default: int, *, minimum: int = 1) -> int: + raw = environ.get(name) + if raw is None or not str(raw).strip(): + return default + try: + value = int(str(raw).strip()) + except ValueError as exc: + raise ValueError(f"{name} must be an integer, got {raw!r}") from exc + if value < minimum: + raise ValueError(f"{name} must be >= {minimum}, got {value}") + return value + + +def _float(environ: Mapping[str, str], name: str, default: float, *, minimum: float) -> float: + raw = environ.get(name) + if raw is None or not str(raw).strip(): + return default + try: + value = float(str(raw).strip()) + except ValueError as exc: + raise ValueError(f"{name} must be a number, got {raw!r}") from exc + if value < minimum: + raise ValueError(f"{name} must be >= {minimum}, got {value}") + return value + + +@dataclass(frozen=True) +class BudgetSettings: + """Knobs for hold estimation, output capping and reservation lifetimes.""" + + #: Global per-call output cap (tokens); the injected ``max_tokens`` never exceeds it. + output_token_ceiling: int = 8192 + #: Below this affordable output cap a call is refused instead of sent. + min_output_tokens: int = 256 + #: Multiplier applied to the byte-based prompt estimate. + input_estimate_safety: float = 1.5 + #: Estimator divisor (UTF-8 bytes per token; 3 is conservative for code). + input_bytes_per_token: float = 3.0 + #: Estimator per-message overhead (role/format tokens). + per_message_token_overhead: int = 4 + #: A hold older than this is EXPIRED (charged in full) by the next reservation. + reservation_deadline_seconds: int = 900 + #: Wall-clock cap on one streamed response; must stay below the deadline minus the upstream timeout. + stream_total_timeout_seconds: int = 600 + #: Lifetime of an inference capability handed to Goose (read once per process). + capability_ttl_seconds: int = 7 * 24 * 3600 + + @classmethod + def from_env(cls, environ: Optional[Mapping[str, str]] = None) -> "BudgetSettings": + env = os.environ if environ is None else environ + settings = cls( + output_token_ceiling=_int(env, "INFERENCE_OUTPUT_TOKEN_CEILING", cls.output_token_ceiling), + min_output_tokens=_int(env, "INFERENCE_MIN_OUTPUT_TOKENS", cls.min_output_tokens), + input_estimate_safety=_float( + env, "INFERENCE_INPUT_ESTIMATE_SAFETY", cls.input_estimate_safety, minimum=1.0 + ), + input_bytes_per_token=_float( + env, "INFERENCE_INPUT_BYTES_PER_TOKEN", cls.input_bytes_per_token, minimum=0.5 + ), + per_message_token_overhead=_int( + env, "INFERENCE_PER_MESSAGE_TOKEN_OVERHEAD", cls.per_message_token_overhead, minimum=0 + ), + reservation_deadline_seconds=_int( + env, "INFERENCE_RESERVATION_DEADLINE_SECONDS", cls.reservation_deadline_seconds + ), + stream_total_timeout_seconds=_int( + env, "INFERENCE_STREAM_TOTAL_TIMEOUT_SECONDS", cls.stream_total_timeout_seconds + ), + capability_ttl_seconds=_int( + env, "INFERENCE_CAPABILITY_TTL_SECONDS", cls.capability_ttl_seconds + ), + ) + if settings.min_output_tokens > settings.output_token_ceiling: + raise ValueError( + "INFERENCE_MIN_OUTPUT_TOKENS must not exceed INFERENCE_OUTPUT_TOKEN_CEILING" + ) + return settings diff --git a/src/research/budget/study_policy.py b/src/research/budget/study_policy.py new file mode 100644 index 0000000..2d5586c --- /dev/null +++ b/src/research/budget/study_policy.py @@ -0,0 +1,93 @@ +"""Study-level budget policy helpers shared by the study and budget routers.""" + +from __future__ import annotations + +import uuid +from datetime import datetime +from decimal import Decimal +from typing import Any, Optional + +from sqlalchemy import select + +from database.research_schemas import StudyAgentProfile +from research.study.agents.enums import METERED_FRAMEWORKS + +from .errors import PriceMissing +from .pricing import MICRO_USD_PER_USD, get_model_price + +__all__ = [ + "budget_policy_payload", + "metered_selections", + "micro_to_usd_string", + "missing_prices", +] + + +def micro_to_usd_string(micro_usd: Optional[int]) -> Optional[str]: + if micro_usd is None: + return None + return f"{Decimal(int(micro_usd)) / MICRO_USD_PER_USD:.2f}" + + +def metered_selections(db: Any, study_id: uuid.UUID) -> list[dict[str, Any]]: + """The study's frozen arms whose runtime spends from the shared key.""" + rows = db.execute( + select(StudyAgentProfile) + .where(StudyAgentProfile.study_id == study_id) + .order_by(StudyAgentProfile.selection_order.asc()) + ).scalars().all() + selections = [] + for row in rows: + snapshot = row.profile_snapshot_json or {} + framework = str(snapshot.get("framework_version") or "").strip().lower() + if framework not in METERED_FRAMEWORKS: + continue + connection_id = snapshot.get("connection_id") + selections.append( + { + "profile_id": str(row.profile_id), + "name": snapshot.get("name", ""), + "model": snapshot.get("model", ""), + "framework_version": framework, + "connection_id": str(connection_id) if connection_id else None, + } + ) + return selections + + +def missing_prices(db: Any, selections: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Metered arms whose frozen model has no price row (they would fail closed).""" + missing = [] + for selection in selections: + connection_id = selection.get("connection_id") + try: + get_model_price( + db, + uuid.UUID(str(connection_id)) if connection_id else None, + str(selection.get("model") or ""), + ) + except (PriceMissing, ValueError): + missing.append( + { + "profile_id": selection.get("profile_id"), + "name": selection.get("name"), + "model": selection.get("model"), + "connection_id": connection_id, + } + ) + return missing + + +def budget_policy_payload(study: Any, selections: list[dict[str, Any]]) -> dict[str, Any]: + updated_at = getattr(study, "inference_budget_updated_at", None) + fraction = getattr(study, "inference_budget_warning_fraction", None) + default = int(getattr(study, "inference_budget_default_micro_usd", 0) or 0) + return { + "metered": bool(selections), + "metered_profile_ids": [selection["profile_id"] for selection in selections], + "default_budget_micro_usd": default, + "default_budget_usd": micro_to_usd_string(default), + "warning_fraction": float(fraction) if fraction is not None else 0.8, + "updated_at": updated_at.isoformat() if isinstance(updated_at, datetime) else None, + "updated_by": getattr(study, "inference_budget_updated_by", None), + } diff --git a/src/research/budget/usage.py b/src/research/budget/usage.py new file mode 100644 index 0000000..bfa8222 --- /dev/null +++ b/src/research/budget/usage.py @@ -0,0 +1,118 @@ +"""Usage extraction that is safe for OpenAI *and* OpenRouter shapes. + +OpenAI's ``stream_options.include_usage`` sends a final chunk with +``choices: []`` and a ``usage`` object; OpenRouter's final chunk carries +``usage`` (with ``cost``) next to a choice with an empty delta. The rule here +is shape-agnostic: the last chunk whose ``usage`` is an object wins. An error +object inside a 200 stream (OpenRouter) or a body without usage yields +``None`` so the caller forfeits the hold instead of guessing. +""" + +from __future__ import annotations + +import json +from decimal import Decimal, InvalidOperation +from typing import Any, Optional + +from .models import UsageSnapshot + +__all__ = ["extract_usage_from_json", "extract_usage_from_sse", "usage_from_object"] + + +def _int_or_none(value: Any) -> Optional[int]: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)) and value >= 0: + return int(value) + return None + + +def _decimal_or_none(value: Any) -> Optional[Decimal]: + if value is None or isinstance(value, bool): + return None + try: + amount = Decimal(str(value)) + except InvalidOperation: + return None + if not amount.is_finite() or amount < 0: + return None + return amount + + +def usage_from_object(usage: Any, *, finish_reason: Optional[str] = None) -> Optional[UsageSnapshot]: + if not isinstance(usage, dict): + return None + prompt = _int_or_none(usage.get("prompt_tokens")) + completion = _int_or_none(usage.get("completion_tokens")) + if prompt is None and completion is None: + return None + details = usage.get("prompt_tokens_details") + cached = _int_or_none(details.get("cached_tokens")) if isinstance(details, dict) else None + return UsageSnapshot( + prompt_tokens=prompt, + completion_tokens=completion, + cached_prompt_tokens=cached, + cost_usd=_decimal_or_none(usage.get("cost")), + finish_reason=finish_reason, + ) + + +def _finish_reason(payload: dict) -> Optional[str]: + choices = payload.get("choices") + if isinstance(choices, list): + for choice in choices: + if isinstance(choice, dict) and choice.get("finish_reason"): + return str(choice["finish_reason"]) + return None + + +def extract_usage_from_json(body: Any) -> Optional[UsageSnapshot]: + """Usage from a non-streaming Chat Completions body (``None`` when absent).""" + if not isinstance(body, dict): + return None + if "error" in body and "usage" not in body: + return None + return usage_from_object(body.get("usage"), finish_reason=_finish_reason(body)) + + +def extract_usage_from_sse(raw_sse: str) -> Optional[UsageSnapshot]: + """Usage from a buffered SSE stream; the last usage-bearing chunk wins.""" + if not raw_sse: + return None + usage: Optional[UsageSnapshot] = None + finish_reason: Optional[str] = None + saw_error = False + for line in raw_sse.splitlines(): + line = line.strip() + if not line.startswith("data:"): + continue + data = line[len("data:"):].strip() + if not data or data == "[DONE]": + continue + try: + chunk = json.loads(data) + except ValueError: + continue + if not isinstance(chunk, dict): + continue + if isinstance(chunk.get("error"), dict): + saw_error = True + reason = _finish_reason(chunk) + if reason: + finish_reason = reason + found = usage_from_object(chunk.get("usage"), finish_reason=finish_reason) + if found is not None: + usage = found + if usage is None: + return None + if saw_error and not usage.has_tokens: + return None + if usage.finish_reason is None and finish_reason is not None: + usage = UsageSnapshot( + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + cached_prompt_tokens=usage.cached_prompt_tokens, + cost_usd=usage.cost_usd, + finish_reason=finish_reason, + ) + return usage diff --git a/src/research/canonical.py b/src/research/canonical.py new file mode 100644 index 0000000..69684ca --- /dev/null +++ b/src/research/canonical.py @@ -0,0 +1,55 @@ +"""Deterministic canonical JSON and SHA-256 hashing (shared). + +Equivalent documents must hash identically regardless of dictionary insertion +order or non-semantic whitespace. We achieve that by always serializing with +``sort_keys=True`` and compact separators over the *parsed* structure, so an +input string with incidental whitespace collapses to the same canonical bytes. + +This module holds the backend-agnostic primitives shared by every research +artifact (Issue 01 receipts, Issue 02 study protocols, and later issues). The +compatibility package re-exports these names so existing imports keep working. +""" + +from __future__ import annotations + +import hashlib +import json +import uuid +from datetime import date, datetime +from enum import Enum +from typing import Any + + +def _json_default(value: Any) -> Any: + """Coerce common non-JSON types to a stable representation.""" + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, Enum): + return value.value + if hasattr(value, "model_dump"): + # Pydantic models nested inside plain dicts. + return value.model_dump(mode="json") + raise TypeError(f"Object of type {type(value).__name__} is not canonical-JSON serializable") + + +def canonical_json(obj: Any) -> str: + """Return the canonical JSON string for ``obj`` (sorted keys, compact).""" + return json.dumps( + obj, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + default=_json_default, + ) + + +def canonical_bytes(obj: Any) -> bytes: + """UTF-8 encoded :func:`canonical_json`.""" + return canonical_json(obj).encode("utf-8") + + +def canonical_hash(obj: Any) -> str: + """SHA-256 hex digest of :func:`canonical_bytes`.""" + return hashlib.sha256(canonical_bytes(obj)).hexdigest() diff --git a/src/research/compatibility/__init__.py b/src/research/compatibility/__init__.py new file mode 100644 index 0000000..b58daa4 --- /dev/null +++ b/src/research/compatibility/__init__.py @@ -0,0 +1,20 @@ +"""ACP host-evidence and compatibility gate (Issue 01). + +Public surface: + +* :mod:`research.compatibility.enums` - closed vocabularies for capabilities, + states, fidelity, enforcement owners, decisions and reason codes. +* :mod:`research.compatibility.models` - Pydantic v2 contracts for the + ``AcpCapabilityReceiptV1`` (and its versioned sub-objects). +* :mod:`research.compatibility.canonical` - deterministic canonical JSON and + SHA-256 hashing. +* :mod:`research.compatibility.redaction` - recursive secret removal. +* :mod:`research.compatibility.receipt` - fixture-transcript ingestion and + receipt validation. +* :mod:`research.compatibility.evaluate` - the deterministic compatibility + evaluator. + +The persistence adapters live in :mod:`research.compatibility.store` and take +a caller-supplied SQLAlchemy ``Session`` so this package stays free of any +application/App import. +""" diff --git a/src/research/compatibility/canonical.py b/src/research/compatibility/canonical.py new file mode 100644 index 0000000..33aaba9 --- /dev/null +++ b/src/research/compatibility/canonical.py @@ -0,0 +1,41 @@ +"""Deterministic canonical JSON and SHA-256 hashing (Issue 01 surface). + +The generic primitives live in :mod:`research.canonical` so every research +artifact (receipts, study protocols, ...) shares one serializer. This module +re-exports them under their historical names and keeps the receipt-specific +``content_hash`` helper that excludes a receipt's own hash field. +""" + +from __future__ import annotations + +from typing import Any + +from research.canonical import ( # noqa: F401 - intentional re-export + canonical_bytes, + canonical_hash, + canonical_json, +) + +# ``content_hash`` is the only field excluded from its own digest. Keeping the +# name in one place avoids a silent drift between the builder and the verifier. +CONTENT_HASH_FIELD = "content_hash" + + +def _dump_without_hash(receipt: Any) -> dict[str, Any]: + if hasattr(receipt, "model_dump"): + data = receipt.model_dump(mode="json") + elif isinstance(receipt, dict): + data = dict(receipt) + else: + raise TypeError("receipt must be a Pydantic model or a mapping") + data.pop(CONTENT_HASH_FIELD, None) + return data + + +def receipt_content_hash(receipt_without_hash: Any) -> str: + """Hash a receipt (model or mapping) ignoring its own ``content_hash``. + + The argument may either already have ``content_hash`` set (in which case it + is stripped before hashing) or be a bare dict/model in its pre-hash form. + """ + return canonical_hash(_dump_without_hash(receipt_without_hash)) diff --git a/src/research/compatibility/enums.py b/src/research/compatibility/enums.py new file mode 100644 index 0000000..9676087 --- /dev/null +++ b/src/research/compatibility/enums.py @@ -0,0 +1,99 @@ +"""Closed vocabularies for the ACP compatibility gate. + +Every value here is part of a persisted contract (receipt JSON, decision +responses, reason codes), so members are additive only: renaming or removing a +member is a breaking change to already-stored receipts. +""" + +from __future__ import annotations + +from enum import Enum + + +class CapabilityId(str, Enum): + """Host/agent capability observed by a pinned fixture probe.""" + + INITIALIZE = "INITIALIZE" + TOOL_LIFECYCLE = "TOOL_LIFECYCLE" + TOOL_UPDATES = "TOOL_UPDATES" + PERMISSION_REQUEST = "PERMISSION_REQUEST" + EDIT_PROPOSAL = "EDIT_PROPOSAL" + DIFF = "DIFF" + CANCELLATION = "CANCELLATION" + SESSION_CLOSE = "SESSION_CLOSE" + SESSION_LOAD = "SESSION_LOAD" + PLANS = "PLANS" + USAGE = "USAGE" + + +class CapabilityState(str, Enum): + """Observed support state for one capability. + + ``UNKNOWN``, ``UNAVAILABLE``, ``PARTIAL`` and ``BROKEN`` are deliberately + distinct from ``SUPPORTED``: an ambiguous or absent observation must never + be promoted to a supported claim. + """ + + SUPPORTED = "SUPPORTED" + PARTIAL = "PARTIAL" + UNKNOWN = "UNKNOWN" + UNAVAILABLE = "UNAVAILABLE" + BROKEN = "BROKEN" + + +class Fidelity(str, Enum): + """How faithfully the observation captures the underlying behavior.""" + + EXACT = "EXACT" + NORMALIZED = "NORMALIZED" + INFERRED = "INFERRED" + + +class EnforcementOwner(str, Enum): + """Which component actually enforces the observed capability.""" + + HOST = "HOST" + AGENT = "AGENT" + PROXY = "PROXY" + PLUGIN = "PLUGIN" + UNKNOWN = "UNKNOWN" + + +class CompatibilityDecision(str, Enum): + """Terminal outcome of evaluating required capabilities.""" + + COMPATIBLE = "COMPATIBLE" + INCOMPATIBLE_ENVIRONMENT = "INCOMPATIBLE_ENVIRONMENT" + INSUFFICIENT_EVIDENCE = "INSUFFICIENT_EVIDENCE" + BROKEN = "BROKEN" + + +class CompatibilityReasonCode(str, Enum): + """Machine-readable justification for a compatibility decision.""" + + OK = "OK" + + # Receipt presence / integrity. + MISSING_RECEIPT = "MISSING_RECEIPT" + RECEIPT_UNVERIFIED = "RECEIPT_UNVERIFIED" + RECEIPT_HASH_MISMATCH = "RECEIPT_HASH_MISMATCH" + + # Environment / identity mismatches. + ENVIRONMENT_MISMATCH = "ENVIRONMENT_MISMATCH" + IDE_BUILD_MISMATCH = "IDE_BUILD_MISMATCH" + AI_ASSISTANT_BUILD_MISMATCH = "AI_ASSISTANT_BUILD_MISMATCH" + PLUGIN_VERSION_MISMATCH = "PLUGIN_VERSION_MISMATCH" + AGENT_ID_MISMATCH = "AGENT_ID_MISMATCH" + AGENT_VERSION_MISMATCH = "AGENT_VERSION_MISMATCH" + ADAPTER_VERSION_MISMATCH = "ADAPTER_VERSION_MISMATCH" + PROTOCOL_VERSION_MISMATCH = "PROTOCOL_VERSION_MISMATCH" + + # Capability evidence gaps. + CAPABILITY_MISSING = "CAPABILITY_MISSING" + CAPABILITY_UNKNOWN = "CAPABILITY_UNKNOWN" + CAPABILITY_PARTIAL = "CAPABILITY_PARTIAL" + CAPABILITY_UNAVAILABLE = "CAPABILITY_UNAVAILABLE" + CAPABILITY_BROKEN = "CAPABILITY_BROKEN" + + # Fixture parsing. + PARSE_FAILED = "PARSE_FAILED" diff --git a/src/research/compatibility/evaluate.py b/src/research/compatibility/evaluate.py new file mode 100644 index 0000000..3fba1f5 --- /dev/null +++ b/src/research/compatibility/evaluate.py @@ -0,0 +1,325 @@ +"""Deterministic compatibility evaluator. + +Every rule below is explicit and order-independent except for the aggregate +severity ranking (``BROKEN`` > ``INCOMPATIBLE_ENVIRONMENT`` > +``INSUFFICIENT_EVIDENCE`` > ``COMPATIBLE``). Evaluation never silently passes a +missing, partial, or unverified requirement. +""" + +from __future__ import annotations + +from typing import Optional + +from .canonical import receipt_content_hash +from .enums import ( + CapabilityState, + CompatibilityDecision, + CompatibilityReasonCode, +) +from .models import ( + AcpCapabilityReceiptV1, + AgentIdentity, + CapabilityEvaluation, + CompatibilityRequest, + CompatibilityResult, + EnvironmentTuple, + ReasonDetail, + RequiredCapability, +) +from .receipt import validate_receipt + +_ENVIRONMENT_FIELD_REASONS = { + "ide_build": CompatibilityReasonCode.IDE_BUILD_MISMATCH, + "ai_assistant_build": CompatibilityReasonCode.AI_ASSISTANT_BUILD_MISMATCH, + "plugin_version": CompatibilityReasonCode.PLUGIN_VERSION_MISMATCH, + "os": CompatibilityReasonCode.ENVIRONMENT_MISMATCH, + "arch": CompatibilityReasonCode.ENVIRONMENT_MISMATCH, + "host_kind": CompatibilityReasonCode.ENVIRONMENT_MISMATCH, +} + +_AGENT_FIELD_REASONS = { + "agent_id": CompatibilityReasonCode.AGENT_ID_MISMATCH, + "agent_version": CompatibilityReasonCode.AGENT_VERSION_MISMATCH, + "adapter_version": CompatibilityReasonCode.ADAPTER_VERSION_MISMATCH, +} + +# Aggregate severity ranking (higher wins). +_SEVERITY = { + CompatibilityDecision.COMPATIBLE: 0, + CompatibilityDecision.INSUFFICIENT_EVIDENCE: 1, + CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT: 2, + CompatibilityDecision.BROKEN: 3, +} + +_REASON_SEVERITY = { + CompatibilityReasonCode.OK: CompatibilityDecision.COMPATIBLE, + CompatibilityReasonCode.MISSING_RECEIPT: CompatibilityDecision.INSUFFICIENT_EVIDENCE, + CompatibilityReasonCode.RECEIPT_UNVERIFIED: CompatibilityDecision.INSUFFICIENT_EVIDENCE, + CompatibilityReasonCode.RECEIPT_HASH_MISMATCH: CompatibilityDecision.INSUFFICIENT_EVIDENCE, + CompatibilityReasonCode.ENVIRONMENT_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.IDE_BUILD_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.AI_ASSISTANT_BUILD_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.PLUGIN_VERSION_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.AGENT_ID_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.AGENT_VERSION_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.ADAPTER_VERSION_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.PROTOCOL_VERSION_MISMATCH: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.CAPABILITY_MISSING: CompatibilityDecision.INSUFFICIENT_EVIDENCE, + CompatibilityReasonCode.CAPABILITY_UNKNOWN: CompatibilityDecision.INSUFFICIENT_EVIDENCE, + CompatibilityReasonCode.CAPABILITY_PARTIAL: CompatibilityDecision.INSUFFICIENT_EVIDENCE, + CompatibilityReasonCode.CAPABILITY_UNAVAILABLE: CompatibilityDecision.INCOMPATIBLE_ENVIRONMENT, + CompatibilityReasonCode.CAPABILITY_BROKEN: CompatibilityDecision.BROKEN, + CompatibilityReasonCode.PARSE_FAILED: CompatibilityDecision.BROKEN, +} + + +def _result( + reasons: list[ReasonDetail], + per_capability: Optional[list[CapabilityEvaluation]] = None, +) -> CompatibilityResult: + if not reasons: + return CompatibilityResult( + decision=CompatibilityDecision.COMPATIBLE, + reasons=[ReasonDetail(code=CompatibilityReasonCode.OK, message="all requirements satisfied")], + per_capability=per_capability or [], + ) + decision = CompatibilityDecision.COMPATIBLE + for reason in reasons: + candidate = _REASON_SEVERITY.get( + reason.code, CompatibilityDecision.INSUFFICIENT_EVIDENCE + ) + if _SEVERITY[candidate] > _SEVERITY[decision]: + decision = candidate + return CompatibilityResult( + decision=decision, + reasons=reasons, + per_capability=per_capability or [], + ) + + +def _environment_reasons( + expected: Optional[EnvironmentTuple], actual: EnvironmentTuple +) -> list[ReasonDetail]: + if expected is None: + return [] + reasons: list[ReasonDetail] = [] + for field, code in _ENVIRONMENT_FIELD_REASONS.items(): + expected_value = getattr(expected, field, None) + if expected_value is None: + continue + actual_value = getattr(actual, field, None) + if actual_value != expected_value: + reasons.append( + ReasonDetail( + code=code, + message=( + f"environment.{field} expected {expected_value!r} " + f"but receipt has {actual_value!r}" + ), + ) + ) + return reasons + + +def _agent_reasons( + expected: Optional[AgentIdentity], actual: AgentIdentity +) -> list[ReasonDetail]: + if expected is None: + return [] + reasons: list[ReasonDetail] = [] + for field, code in _AGENT_FIELD_REASONS.items(): + expected_value = getattr(expected, field, None) + if expected_value is None: + continue + actual_value = getattr(actual, field, None) + if actual_value != expected_value: + reasons.append( + ReasonDetail( + code=code, + message=( + f"agent.{field} expected {expected_value!r} " + f"but receipt has {actual_value!r}" + ), + ) + ) + return reasons + + +def _evaluate_requirement( + receipt: AcpCapabilityReceiptV1, requirement: RequiredCapability +) -> CapabilityEvaluation: + observed = next( + ( + operation + for operation in receipt.observed_operations + if operation.capability == requirement.capability + ), + None, + ) + base = CapabilityEvaluation( + capability=requirement.capability, + required_state=requirement.require_state, + observed_state=observed.state if observed else None, + enforcement_owner=observed.enforcement_owner if observed else None, + satisfied=False, + ) + + if observed is None: + base.reason = ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_MISSING, + capability=requirement.capability, + message=( + f"receipt has no observed operation for {requirement.capability.value}" + ), + ) + return base + + state = observed.state + if state == CapabilityState.BROKEN: + base.reason = ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_BROKEN, + capability=requirement.capability, + message=f"{requirement.capability.value} is broken in this environment", + ) + return base + + if state == CapabilityState.UNAVAILABLE: + base.reason = ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_UNAVAILABLE, + capability=requirement.capability, + message=f"{requirement.capability.value} is unavailable in this environment", + ) + return base + + if state == CapabilityState.UNKNOWN: + if requirement.require_state == CapabilityState.UNKNOWN: + base.satisfied = True + return base + base.reason = ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_UNKNOWN, + capability=requirement.capability, + message=f"{requirement.capability.value} was never observed (UNKNOWN)", + ) + return base + + if state == CapabilityState.PARTIAL: + if requirement.require_state in ( + CapabilityState.PARTIAL, + CapabilityState.UNKNOWN, + ): + base.satisfied = True + return base + base.reason = ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_PARTIAL, + capability=requirement.capability, + message=( + f"{requirement.capability.value} is only PARTIAL but " + f"{requirement.require_state.value} was required" + ), + ) + return base + + # Remaining state: SUPPORTED. + if requirement.require_state == CapabilityState.SUPPORTED: + base.satisfied = True + return base + + # require_state is PARTIAL or UNKNOWN, both weaker than SUPPORTED. + base.satisfied = True + return base + + +def evaluate_compatibility(request: CompatibilityRequest) -> CompatibilityResult: + """Evaluate ``request`` and return a typed, justified decision.""" + receipt = request.receipt + if receipt is None: + return _result( + [ + ReasonDetail( + code=CompatibilityReasonCode.MISSING_RECEIPT, + message="no capability receipt was supplied", + ) + ] + ) + + if not isinstance(receipt, AcpCapabilityReceiptV1): + try: + receipt = AcpCapabilityReceiptV1.model_validate(receipt) + except Exception as error: # noqa: BLE001 - report any parse failure + return _result( + [ + ReasonDetail( + code=CompatibilityReasonCode.RECEIPT_UNVERIFIED, + message=f"receipt could not be parsed: {error}", + ) + ] + ) + + validation_reasons = validate_receipt(receipt) + if validation_reasons: + return _result( + [ + ReasonDetail( + code=CompatibilityReasonCode.RECEIPT_UNVERIFIED, + capability=reason.capability, + message=f"receipt is not scoped: {reason.message}", + ) + for reason in validation_reasons + ] + ) + + broken = receipt.status == CapabilityState.BROKEN or any( + operation.state == CapabilityState.BROKEN + for operation in receipt.observed_operations + ) + if broken: + if receipt.status == CapabilityState.BROKEN and not receipt.observed_operations: + reason = ReasonDetail( + code=CompatibilityReasonCode.PARSE_FAILED, + message="fixture transcript could not be parsed", + ) + else: + reason = ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_BROKEN, + message="receipt contains a broken capability", + ) + return _result([reason]) + + if receipt_content_hash(receipt) != receipt.content_hash: + return _result( + [ + ReasonDetail( + code=CompatibilityReasonCode.RECEIPT_HASH_MISMATCH, + message="receipt content hash does not match its canonical content", + ) + ] + ) + + reasons: list[ReasonDetail] = [] + reasons.extend(_environment_reasons(request.expected_environment, receipt.environment)) + reasons.extend(_agent_reasons(request.expected_agent, receipt.agent)) + if ( + request.expected_protocol_version is not None + and request.expected_protocol_version != receipt.protocol_version + ): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.PROTOCOL_VERSION_MISMATCH, + message=( + f"protocol_version expected {request.expected_protocol_version!r} " + f"but receipt has {receipt.protocol_version!r}" + ), + ) + ) + + per_capability = [ + _evaluate_requirement(receipt, requirement) + for requirement in request.requirements + ] + reasons.extend( + evaluation.reason + for evaluation in per_capability + if evaluation.reason is not None + ) + + return _result(reasons, per_capability) diff --git a/src/research/compatibility/models.py b/src/research/compatibility/models.py new file mode 100644 index 0000000..f9c71d1 --- /dev/null +++ b/src/research/compatibility/models.py @@ -0,0 +1,169 @@ +"""Pydantic v2 contracts for the ACP capability receipt and evaluation API. + +These models are the wire + persistence contract for Issue 01. They are +deliberately permissive about *missing* optional evidence (so an incomplete +receipt can be represented and then explicitly rejected by +:func:`research.compatibility.receipt.validate_receipt`) but strict about +enum-typed values. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import ( + CapabilityId, + CapabilityState, + CompatibilityDecision, + CompatibilityReasonCode, + EnforcementOwner, + Fidelity, +) + + +class EnvironmentTuple(BaseModel): + """Version-pinned host environment a receipt was captured against. + + Fields are optional so the same type can express a *partial* expectation in + :class:`CompatibilityRequest` (only present fields are compared). On a + receipt, :func:`research.compatibility.receipt.validate_receipt` requires + the core fields to be present. + """ + + model_config = ConfigDict(extra="ignore") + + ide_build: Optional[str] = None + ai_assistant_build: Optional[str] = None + plugin_version: Optional[str] = None + os: Optional[str] = None + arch: Optional[str] = None + host_kind: Optional[str] = None + + +class AgentIdentity(BaseModel): + """Agent + adapter identity a receipt was captured against. + + ``agent_id`` / ``agent_version`` are required on a receipt (enforced by + validation); they are optional here so a partial expected identity can be + supplied as an override. + """ + + model_config = ConfigDict(extra="ignore") + + agent_id: Optional[str] = None + agent_version: Optional[str] = None + adapter_version: Optional[str] = None + release_digest: Optional[str] = None + executable_digest: Optional[str] = None + + +class EvidenceRef(BaseModel): + """A content-addressed pointer to redacted evidence. + + The raw payload is never stored here; only its digest and a stable + reference are. ``redacted`` records that the pointed-to artifact has been + through :func:`research.compatibility.redaction.redact`. + """ + + model_config = ConfigDict(extra="ignore") + + ref: str + kind: str + sha256: str + size: Optional[int] = None + redacted: bool = True + + +class CapabilityResult(BaseModel): + """Observed result for one capability in one environment.""" + + model_config = ConfigDict(extra="ignore") + + capability: CapabilityId + state: CapabilityState + fidelity: Fidelity = Fidelity.NORMALIZED + enforcement_owner: EnforcementOwner = EnforcementOwner.UNKNOWN + limitations: list[str] = Field(default_factory=list) + observed_at: Optional[datetime] = None + evidence_refs: list[str] = Field(default_factory=list) + + +class AcpCapabilityReceiptV1(BaseModel): + """Immutable, redacted, hashed capability receipt (schema version 1).""" + + model_config = ConfigDict(extra="ignore") + + schema_version: str = "1" + receipt_id: UUID + captured_at: datetime + environment: EnvironmentTuple + agent: AgentIdentity + protocol_version: str + declared: dict[str, str | bool] = Field(default_factory=dict) + observed_operations: list[CapabilityResult] = Field(default_factory=list) + evidence: list[EvidenceRef] = Field(default_factory=list) + status: CapabilityState + # Content hash over the canonical serialization of this model with + # ``content_hash`` excluded. Empty until finalised by the builder. + content_hash: str = "" + + +class RequiredCapability(BaseModel): + """One capability predicate a caller requires of a receipt.""" + + model_config = ConfigDict(extra="ignore") + + capability: CapabilityId + require_state: CapabilityState = CapabilityState.SUPPORTED + enforcement_owner: Optional[EnforcementOwner] = None + + +class CompatibilityRequest(BaseModel): + """Evaluate ``requirements`` against a receipt (inline or by id).""" + + model_config = ConfigDict(extra="ignore") + + receipt: Optional[AcpCapabilityReceiptV1] = None + receipt_id: Optional[UUID] = None + requirements: list[RequiredCapability] = Field(default_factory=list) + expected_environment: Optional[EnvironmentTuple] = None + expected_agent: Optional[AgentIdentity] = None + # Negotiated ACP wire protocol the caller requires. + expected_protocol_version: Optional[str] = None + + +class ReasonDetail(BaseModel): + """Typed explanation for one part of a compatibility decision.""" + + model_config = ConfigDict(extra="ignore") + + code: CompatibilityReasonCode + capability: Optional[CapabilityId] = None + message: str + + +class CapabilityEvaluation(BaseModel): + """Per-requirement outcome, independent of the aggregate decision.""" + + model_config = ConfigDict(extra="ignore") + + capability: CapabilityId + required_state: CapabilityState + observed_state: Optional[CapabilityState] = None + enforcement_owner: Optional[EnforcementOwner] = None + satisfied: bool + reason: Optional[ReasonDetail] = None + + +class CompatibilityResult(BaseModel): + """Aggregate compatibility decision and its full justification.""" + + model_config = ConfigDict(extra="ignore") + + decision: CompatibilityDecision + reasons: list[ReasonDetail] = Field(default_factory=list) + per_capability: list[CapabilityEvaluation] = Field(default_factory=list) diff --git a/src/research/compatibility/receipt.py b/src/research/compatibility/receipt.py new file mode 100644 index 0000000..9bd4d0e --- /dev/null +++ b/src/research/compatibility/receipt.py @@ -0,0 +1,444 @@ +"""Fixture-transcript ingestion and receipt validation. + +The ingestion path turns a captured ACP observation transcript into a redacted, +hashed :class:`~research.compatibility.models.AcpCapabilityReceiptV1`. Raw +payloads are never embedded in the receipt: only redacted digests are, so a +receipt remains safe to persist and export. + +A malformed transcript never fails open. It yields a ``BROKEN`` receipt and a +separate ``acp_parse_failed`` system event built by +:func:`parse_failure_event`. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Mapping, Optional +from uuid import uuid4 + +from pydantic import BaseModel + +from .canonical import canonical_bytes, canonical_hash, receipt_content_hash +from .enums import ( + CapabilityId, + CapabilityState, + CompatibilityReasonCode, + EnforcementOwner, + Fidelity, +) +from .models import ( + AcpCapabilityReceiptV1, + AgentIdentity, + CapabilityResult, + EnvironmentTuple, + EvidenceRef, + ReasonDetail, +) +from .redaction import redact + +# Event contract for a failed fixture parse. Kept as a plain dict so the +# canonical research event envelope stays the single event authority (no raw content). +ACP_PARSE_FAILED_EVENT_TYPE = "system.acp_parse_failed" + +_REQUIRED_ENVIRONMENT_FIELDS = ( + "ide_build", + "ai_assistant_build", + "plugin_version", + "os", + "arch", +) + + +def _coerce_fidelity(value: Any) -> Fidelity: + if value is None: + return Fidelity.NORMALIZED + try: + return Fidelity(str(value).strip().upper()) + except ValueError: + return Fidelity.INFERRED + + +def _coerce_owner(value: Any) -> EnforcementOwner: + if value is None: + return EnforcementOwner.UNKNOWN + try: + return EnforcementOwner(str(value).strip().upper()) + except ValueError: + return EnforcementOwner.UNKNOWN + + +def _parse_datetime(value: Any) -> Optional[datetime]: + if value is None or isinstance(value, datetime): + return value + if isinstance(value, str): + text = value.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + return datetime.fromisoformat(text) + except ValueError: + return None + return None + + +def _extract_capabilities(message: Any) -> Optional[Mapping[str, Any]]: + """Pull the capability object out of an initialize request/response.""" + if not isinstance(message, Mapping): + return None + for container_key in ("result", "params"): + container = message.get(container_key) + if isinstance(container, Mapping): + for caps_key in ("agentCapabilities", "clientCapabilities"): + caps = container.get(caps_key) + if isinstance(caps, Mapping): + return caps + for caps_key in ("agentCapabilities", "clientCapabilities"): + caps = message.get(caps_key) + if isinstance(caps, Mapping): + return caps + return None + + +def _flatten_capabilities( + prefix: str, value: Any, out: dict[str, str | bool] +) -> None: + if not isinstance(value, Mapping): + return + for key, item in value.items(): + name = f"{prefix}.{key}" if prefix else str(key) + if isinstance(item, Mapping): + _flatten_capabilities(name, item, out) + elif isinstance(item, bool): + out[name] = item + elif isinstance(item, str): + out[name] = item + elif item is None: + out[name] = False + else: + out[name] = str(item) + + +def _declared_capabilities(transcript: Mapping[str, Any]) -> dict[str, str | bool]: + declared: dict[str, str | bool] = {} + raw = transcript.get("declared") + if isinstance(raw, Mapping): + _flatten_capabilities("", raw, declared) + _flatten_capabilities( + "client", _extract_capabilities(transcript.get("initialize_request")), declared + ) + _flatten_capabilities( + "agent", _extract_capabilities(transcript.get("initialize_response")), declared + ) + return declared + + +def _protocol_version(transcript: Mapping[str, Any]) -> str: + direct = transcript.get("protocol_version") + if isinstance(direct, str) and direct.strip(): + return direct.strip() + for message_key in ("initialize_response", "initialize_request"): + message = transcript.get(message_key) + if not isinstance(message, Mapping): + continue + for container_key in ("result", "params"): + container = message.get(container_key) + if isinstance(container, Mapping): + version = container.get("protocolVersion") + if isinstance(version, str) and version.strip(): + return version.strip() + return "unknown" + + +def _evidence_ref_for_operation( + raw: Mapping[str, Any], + capability: CapabilityId, + index: int, + capture_channel: str, +) -> EvidenceRef: + payload = raw.get("evidence") + if payload is None: + payload = {key: raw.get(key) for key in ("method", "id", "result", "error")} + redacted = redact(payload).value + encoded = canonical_bytes(redacted) + return EvidenceRef( + ref=f"fixture://{capture_channel}/operation/{index}/{capability.value.lower()}", + kind="acp_operation", + sha256=canonical_hash(redacted), + size=len(encoded), + redacted=True, + ) + + +def _redacted_transcript_evidence( + transcript: Any, capture_channel: str +) -> EvidenceRef: + redacted = redact(transcript).value + encoded = canonical_bytes(redacted) + return EvidenceRef( + ref=f"fixture://{capture_channel}/transcript.redacted.json", + kind="redacted_transcript", + sha256=canonical_hash(redacted), + size=len(encoded), + redacted=True, + ) + + +def _parse_operations( + transcript: Any, capture_channel: str +) -> tuple[list[CapabilityResult], list[EvidenceRef], Optional[str]]: + """Parse fixture operations, returning ``(results, evidence, parse_error)``.""" + if not isinstance(transcript, Mapping): + return [], [], "fixture transcript is not an object" + + explicit_error = transcript.get("parse_error") + if explicit_error: + return [], [], str(explicit_error) + + raw_operations = transcript.get("operations") + if raw_operations is None: + raw_operations = [] + if not isinstance(raw_operations, list): + return [], [], "fixture 'operations' must be a list" + + results: list[CapabilityResult] = [] + evidence: list[EvidenceRef] = [] + for index, raw in enumerate(raw_operations): + if not isinstance(raw, Mapping): + return [], [], f"operation {index} is not an object" + capability_raw = raw.get("capability") + state_raw = raw.get("state") + if not capability_raw or not state_raw: + return [], [], f"operation {index} is missing capability or state" + try: + capability = CapabilityId(str(capability_raw).strip().upper()) + except ValueError: + return [], [], f"operation {index} has unknown capability {capability_raw!r}" + try: + state = CapabilityState(str(state_raw).strip().upper()) + except ValueError: + return [], [], f"operation {index} has unknown state {state_raw!r}" + + ref = _evidence_ref_for_operation(raw, capability, index, capture_channel) + evidence.append(ref) + + limitations = raw.get("limitations") or [] + if isinstance(limitations, str): + limitations = [limitations] + elif not isinstance(limitations, list): + limitations = [str(limitations)] + + results.append( + CapabilityResult( + capability=capability, + state=state, + fidelity=_coerce_fidelity(raw.get("fidelity")), + enforcement_owner=_coerce_owner(raw.get("enforcement_owner")), + limitations=[str(item) for item in limitations], + observed_at=_parse_datetime(raw.get("observed_at")), + evidence_refs=[ref.ref], + ) + ) + return results, evidence, None + + +def _aggregate_status(results: list[CapabilityResult]) -> CapabilityState: + if not results: + return CapabilityState.UNKNOWN + states = {result.state for result in results} + for state in ( + CapabilityState.BROKEN, + CapabilityState.UNAVAILABLE, + CapabilityState.UNKNOWN, + CapabilityState.PARTIAL, + ): + if state in states: + return state + return CapabilityState.SUPPORTED + + +def build_receipt_from_fixture( + transcript: dict, + environment: EnvironmentTuple, + agent: AgentIdentity, + *, + capture_channel: str, + captured_at: Optional[datetime] = None, +) -> AcpCapabilityReceiptV1: + """Build a redacted, hashed receipt from a captured fixture transcript. + + A transcript is treated as broken when it carries a ``parse_error`` or is + structurally malformed (unknown capability/state, non-object operations, + or a non-object transcript). In that case the receipt has + ``status=BROKEN``, no observed operations, and only the redacted + transcript digest as evidence. + """ + captured = captured_at or datetime.now(timezone.utc) + operations, operation_evidence, parse_error = _parse_operations( + transcript, capture_channel + ) + + if parse_error is not None: + operations = [] + operation_evidence = [] + + evidence = [_redacted_transcript_evidence(transcript, capture_channel)] + evidence.extend(operation_evidence) + + if parse_error is not None: + status = CapabilityState.BROKEN + else: + status = _aggregate_status(operations) + + declared = _declared_capabilities(transcript) if isinstance(transcript, Mapping) else {} + + receipt = AcpCapabilityReceiptV1( + schema_version="1", + receipt_id=uuid4(), + captured_at=captured, + environment=environment, + agent=agent, + protocol_version=( + _protocol_version(transcript) if isinstance(transcript, Mapping) else "unknown" + ), + declared=declared, + observed_operations=operations, + evidence=evidence, + status=status, + content_hash="", + ) + receipt.content_hash = receipt_content_hash(receipt) + return receipt + + +def receipt_parse_error(transcript: Any) -> Optional[str]: + """Return the parse-error reason a transcript would yield, or ``None``.""" + return _parse_operations(transcript, "fixture")[2] + + +def parse_failure_event( + reason: str, + *, + transcript: Any = None, + capture_channel: str = "fixture", + environment: Optional[EnvironmentTuple] = None, + agent: Optional[AgentIdentity] = None, + captured_at: Optional[datetime] = None, +) -> dict: + """Build the ``system.acp_parse_failed`` event for a broken transcript. + + The event carries only a hashed evidence reference and caller-supplied + metadata; it never fabricates probe content. + """ + captured = captured_at or datetime.now(timezone.utc) + evidence = _redacted_transcript_evidence(transcript, capture_channel) + return { + "event_type": ACP_PARSE_FAILED_EVENT_TYPE, + "source": "acp", + "severity": "error", + "reason": str(reason), + "captured_at": captured.isoformat(), + "environment": environment.model_dump(mode="json") if environment else None, + "agent": agent.model_dump(mode="json") if agent else None, + "evidence": evidence.model_dump(mode="json"), + } + + +def _as_mapping(receipt: AcpCapabilityReceiptV1 | Mapping[str, Any]) -> Mapping[str, Any]: + if isinstance(receipt, BaseModel): + return receipt.model_dump(mode="json") + if isinstance(receipt, Mapping): + return receipt + return {} + + +def _safe_capability(value: Any) -> Optional[CapabilityId]: + if value is None: + return None + try: + return CapabilityId(str(value).strip().upper()) + except ValueError: + return None + + +def validate_receipt( + receipt: AcpCapabilityReceiptV1 | Mapping[str, Any], +) -> list[ReasonDetail]: + """Return the reasons a receipt makes an unscoped capability claim. + + An empty list means the receipt is scoped: it names its environment, + agent identity, negotiated protocol version, and every observed operation + carries both a capability and a state. + """ + data = _as_mapping(receipt) + reasons: list[ReasonDetail] = [] + + environment = data.get("environment") + if not isinstance(environment, Mapping): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.ENVIRONMENT_MISMATCH, + message="receipt environment is missing", + ) + ) + else: + for field in _REQUIRED_ENVIRONMENT_FIELDS: + if not environment.get(field): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.ENVIRONMENT_MISMATCH, + message=f"receipt environment.{field} is missing", + ) + ) + + agent = data.get("agent") + if not isinstance(agent, Mapping) or not agent.get("agent_id"): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.AGENT_ID_MISMATCH, + message="receipt agent.agent_id is missing", + ) + ) + if not isinstance(agent, Mapping) or not agent.get("agent_version"): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.AGENT_VERSION_MISMATCH, + message="receipt agent.agent_version is missing", + ) + ) + + if not data.get("protocol_version"): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.PROTOCOL_VERSION_MISMATCH, + message="receipt protocol_version is missing", + ) + ) + + operations = data.get("observed_operations") + if operations is None: + operations = [] + if not isinstance(operations, list): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.PARSE_FAILED, + message="receipt observed_operations is not a list", + ) + ) + else: + for index, operation in enumerate(operations): + if not isinstance(operation, Mapping) or not operation.get("capability"): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_MISSING, + message=f"observed operation {index} is missing a capability", + ) + ) + elif not operation.get("state"): + reasons.append( + ReasonDetail( + code=CompatibilityReasonCode.CAPABILITY_UNKNOWN, + capability=_safe_capability(operation.get("capability")), + message=f"observed operation {index} is missing a state", + ) + ) + return reasons diff --git a/src/research/compatibility/redaction.py b/src/research/compatibility/redaction.py new file mode 100644 index 0000000..c995a04 --- /dev/null +++ b/src/research/compatibility/redaction.py @@ -0,0 +1,126 @@ +"""Recursive, non-mutating secret redaction. + +Two layers of defence: + +1. **Key-based** - any mapping key whose lower-cased name contains a known + secret marker (``password``, ``token``, ``api_key`` ...) has its whole value + replaced by ``[REDACTED]``. +2. **Value-based** - string scalars are scanned for well-known inline secret + shapes (``Bearer ``, ``sk-...``, ``ghp_...``, ``AKIA...``) even when + they appear under an innocuous key. + +The input is never mutated: only fresh dicts/lists/strings are returned. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Mapping + +REDACTED = "[REDACTED]" + +# Case-insensitive substrings that mark a key as secret-bearing. +SECRET_KEY_PATTERNS: tuple[str, ...] = ( + "password", + "passwd", + "secret", + "token", + "api_key", + "apikey", + "authorization", + "bearer", + "credential", + "private_key", + "access_key", + "client_secret", + "cookie", + "session_token", +) + +# Inline value shapes. The bearer case preserves the scheme for readability. +_VALUE_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = ( + (re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]{4,}"), "Bearer " + REDACTED), + (re.compile(r"\bsk-[A-Za-z0-9_-]{8,}"), REDACTED), + (re.compile(r"\bghp_[A-Za-z0-9]{8,}"), REDACTED), + (re.compile(r"\bAKIA[0-9A-Z]{12,}"), REDACTED), +) + + +@dataclass +class RedactionResult: + """A redacted copy plus counts of what was removed. + + ``value`` is a structurally independent copy; the original object passed to + :func:`redact` is untouched. + """ + + value: Any + key_redactions: int = 0 + value_redactions: int = 0 + paths: list[str] = field(default_factory=list) + + @property + def redaction_count(self) -> int: + return self.key_redactions + self.value_redactions + + @property + def changed(self) -> bool: + return self.redaction_count > 0 + + def to_dict(self) -> dict[str, Any]: + return { + "key_redactions": self.key_redactions, + "value_redactions": self.value_redactions, + "redaction_count": self.redaction_count, + "paths": list(self.paths), + } + + +def is_secret_key(key: Any) -> bool: + """True when a mapping key name marks its value as secret material.""" + lowered = str(key).lower() + return any(pattern in lowered for pattern in SECRET_KEY_PATTERNS) + + +def redact_string(value: str) -> tuple[str, int]: + """Return ``(redacted, count)`` for inline secret shapes in one string.""" + result = value + count = 0 + for pattern, replacement in _VALUE_PATTERNS: + result, substitutions = pattern.subn(replacement, result) + count += substitutions + return result, count + + +def redact(value: Any) -> RedactionResult: + """Recursively redact ``value`` and return a :class:`RedactionResult`.""" + result = RedactionResult(value=None) + result.value = _walk(value, result, path="$") + return result + + +def _walk(value: Any, result: RedactionResult, path: str) -> Any: + if isinstance(value, Mapping): + redacted: dict[Any, Any] = {} + for key, item in value.items(): + child_path = f"{path}.{key}" + if is_secret_key(key): + redacted[key] = REDACTED + result.key_redactions += 1 + result.paths.append(child_path) + else: + redacted[key] = _walk(item, result, child_path) + return redacted + if isinstance(value, (list, tuple)): + return [ + _walk(item, result, f"{path}[{index}]") + for index, item in enumerate(value) + ] + if isinstance(value, str): + redacted_value, count = redact_string(value) + if count: + result.value_redactions += count + result.paths.append(path) + return redacted_value + return value diff --git a/src/research/compatibility/store.py b/src/research/compatibility/store.py new file mode 100644 index 0000000..aa0d958 --- /dev/null +++ b/src/research/compatibility/store.py @@ -0,0 +1,111 @@ +"""CRUD-style persistence helpers for capability receipts. + +These functions take a caller-managed SQLAlchemy ``Session`` so the core +package never imports ``App`` or touches the application singleton. The router +is responsible for session lifecycle (``App.get_db_session`` / +``rollback`` / ``close``). +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional, Sequence + +from sqlalchemy import select + +if TYPE_CHECKING: + import uuid + + from sqlalchemy.orm import Session + +from database.research_schemas import AcpCapabilityReceipt + +from .models import AcpCapabilityReceiptV1 + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def persist_receipt(session: Session, receipt: AcpCapabilityReceiptV1) -> AcpCapabilityReceipt: + """Insert a receipt. + + The evidence digests are part of ``receipt_json.evidence[]`` (the canonical + content), so no child rows are written. Raises + ``sqlalchemy.exc.IntegrityError`` if the ``content_hash`` already exists; the + caller decides whether that is a conflict or an idempotent retry. + """ + environment = receipt.environment + agent = receipt.agent + row = AcpCapabilityReceipt( + receipt_id=receipt.receipt_id, + captured_at=receipt.captured_at, + ide_build=environment.ide_build, + ai_assistant_build=environment.ai_assistant_build, + plugin_version=environment.plugin_version, + os=environment.os, + arch=environment.arch, + agent_id=agent.agent_id, + agent_version=agent.agent_version, + adapter_version=agent.adapter_version, + protocol_version=receipt.protocol_version, + status=receipt.status.value, + content_hash=receipt.content_hash, + receipt_json=receipt.model_dump(mode="json"), + created_at=_now(), + ) + session.add(row) + session.commit() + session.refresh(row) + return row + + +def get_receipt(session: Session, receipt_id: uuid.UUID) -> Optional[AcpCapabilityReceipt]: + """Fetch a receipt row by primary key, or ``None``.""" + return session.get(AcpCapabilityReceipt, receipt_id) + + +def list_receipts( + session: Session, *, limit: int = 100, offset: int = 0 +) -> Sequence[AcpCapabilityReceipt]: + """List receipts newest-first.""" + statement = ( + select(AcpCapabilityReceipt) + .order_by(AcpCapabilityReceipt.captured_at.desc()) + .limit(limit) + .offset(offset) + ) + return list(session.execute(statement).scalars().all()) + + +def row_to_receipt(row: AcpCapabilityReceipt) -> AcpCapabilityReceiptV1: + """Rehydrate the stored receipt JSON into the versioned model.""" + return AcpCapabilityReceiptV1.model_validate(row.receipt_json) + + +def row_to_summary(row: AcpCapabilityReceipt) -> dict[str, Any]: + """Return a compact, non-secret summary safe for list responses.""" + captured_at = row.captured_at + created_at = row.created_at + return { + "receipt_id": str(row.receipt_id), + "captured_at": captured_at.isoformat() if captured_at else None, + "created_at": ( + created_at.isoformat() if isinstance(created_at, datetime) else None + ), + "status": row.status, + "content_hash": row.content_hash, + "protocol_version": row.protocol_version, + "environment": { + "ide_build": row.ide_build, + "ai_assistant_build": row.ai_assistant_build, + "plugin_version": row.plugin_version, + "os": row.os, + "arch": row.arch, + }, + "agent": { + "agent_id": row.agent_id, + "agent_version": row.agent_version, + "adapter_version": row.adapter_version, + }, + } diff --git a/src/research/participants/__init__.py b/src/research/participants/__init__.py new file mode 100644 index 0000000..6cc920c --- /dev/null +++ b/src/research/participants/__init__.py @@ -0,0 +1,14 @@ +"""Participant identity, consent, withdrawal and retention. + +Flat home for the participant identity/consent state machine and the +idempotent retention execution service: + +* :mod:`identity` - the pure enrollment/consent/re-consent/withdrawal state + machine, participant-code generation and retention, the researcher-safe + projection, and the SQLAlchemy persistence adapters. +* :mod:`retention` - pure retention planning/execution, the persistence + helpers plus :class:`SqlAlchemyRetentionStore`, and the bounded retrying + worker entry points. + +The core packages never import ``App``, FastAPI, or a session factory. +""" diff --git a/src/research/participants/enums.py b/src/research/participants/enums.py new file mode 100644 index 0000000..c504529 --- /dev/null +++ b/src/research/participants/enums.py @@ -0,0 +1,116 @@ +"""Closed vocabularies for participant identity and retention. + +Consent is a single acceptance recorded once at join (no document identity or +re-consent), so those vocabularies are gone. A participant withdraws by opting +the account out of data collection (``privacy.collection``), which ends the +enrollment as ``WITHDRAWN``. What remains is the enrollment lifecycle, the +stable rejection reasons, and the retention vocabulary (re-exported from the +study protocol so identity and retention share one published contract). + +Members are part of persisted contracts, so they stay additive. +""" + +from __future__ import annotations + +from enum import Enum + +from research.study.protocol.enums import RetentionAction # noqa: F401 - intentional re-export + +__all__ = [ + "EnrollmentStatus", + "IdentityReasonCode", + "RetentionAction", + "RetentionJobState", + "RetentionReasonCode", + "RetentionState", + "RetentionStepOutcome", +] + + +class EnrollmentStatus(str, Enum): + """Lifecycle of one participant's enrollment in one research study. + + ``ACTIVE`` is the only state in which telemetry may be accepted. A study that + stops marks its enrollments ``STUDY_STOPPED``; an administrative revoke is + ``REVOKED``; a participant who opts out of data collection is ``WITHDRAWN``. + All three are terminal and accept nothing new. + """ + + ACTIVE = "ACTIVE" + REVOKED = "REVOKED" + STUDY_STOPPED = "STUDY_STOPPED" + WITHDRAWN = "WITHDRAWN" + COMPLETED = "COMPLETED" # legacy compatibility for non-research callers + + +class IdentityReasonCode(str, Enum): + """Stable machine-readable reason an identity operation was rejected.""" + + DUPLICATE_ENROLLMENT = "DUPLICATE_ENROLLMENT" + # One active enrollment per account across the platform: another study is + # already live for this participant. + ALREADY_ENROLLED = "ALREADY_ENROLLED" + # A completed study is terminal; the account may join a different study. + REJOIN_NOT_ALLOWED = "REJOIN_NOT_ALLOWED" + INVALID_STATE_TRANSITION = "INVALID_STATE_TRANSITION" + MAPPING_FAILURE = "MAPPING_FAILURE" + UNKNOWN_REVISION = "UNKNOWN_REVISION" + NOT_ACTIVE = "NOT_ACTIVE" + REVOKED = "REVOKED" + + # Eligibility. + ELIGIBLE = "ELIGIBLE" + INELIGIBLE = "INELIGIBLE" + UNKNOWN_ELIGIBILITY = "UNKNOWN_ELIGIBILITY" + + +class RetentionJobState(str, Enum): + """Lifecycle of one idempotent retention job. + + ``PENDING`` is enqueued when a study ends; ``RUNNING`` is written before any + data is touched; ``COMPLETED`` is terminal and never re-executed. A partial + failure is never ``COMPLETED``: it is ``RETRYABLE`` while attempts remain and + ``FAILED`` once the retry budget is exhausted. + """ + + PENDING = "PENDING" + RUNNING = "RUNNING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + RETRYABLE = "RETRYABLE" + + +class RetentionStepOutcome(str, Enum): + """What one planned retention step did (or will do) to one item.""" + + DELETED = "DELETED" + ANONYMIZED = "ANONYMIZED" + RETAINED = "RETAINED" + INVALIDATED = "INVALIDATED" + SKIPPED = "SKIPPED" + FAILED = "FAILED" + + +class RetentionReasonCode(str, Enum): + """Stable machine-readable reason for a job outcome (content-free).""" + + OK = "OK" + ALREADY_COMPLETED = "ALREADY_COMPLETED" + ACTION_MISMATCH = "ACTION_MISMATCH" + PARTIAL_FAILURE = "PARTIAL_FAILURE" + RETRY_EXHAUSTED = "RETRY_EXHAUSTED" + STORE_UNAVAILABLE = "STORE_UNAVAILABLE" + NO_ENROLLMENT = "NO_ENROLLMENT" + + +class RetentionState(str, Enum): + """Persisted per-event retention posture (``ResearchEvent.retention_state``). + + ``ANONYMIZED`` events have account-linkable payload/provenance fields + stripped. ``DELETED`` rows are content-free tombstones that read models must + exclude. + """ + + RETAINED = "RETAINED" + ANONYMIZED = "ANONYMIZED" + DELETED = "DELETED" diff --git a/src/research/participants/erasure.py b/src/research/participants/erasure.py new file mode 100644 index 0000000..bc88e3c --- /dev/null +++ b/src/research/participants/erasure.py @@ -0,0 +1,173 @@ +"""Erase one account's research participation (GDPR Art. 17). + +Hard-deletes everything the research tables hold for the account's participant: +the research events, batch receipts and retention jobs of its enrollments and +sessions, then the participant row itself, which cascades to the enrollments, +sessions, agent runs, assignments and inference-budget rows. Telemetry rows use +``SET NULL`` foreign keys (retention policy, not a parent-row delete, decides +when events disappear), so they are deleted explicitly first: a parent delete +alone would orphan them with their content. + +Every erased enrollment leaves one content-free deletion-ledger record (study, +time and counts; never the account, email or participant code) so researchers +can still account for attrition. + +Like the other persistence helpers here, these functions take a caller-managed +session and never commit: an account erasure is one transaction. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Optional + +from sqlalchemy import delete, func, or_, select + +from database.research_schemas import ( + RECORD_KIND_STUDY_LIFECYCLE, + ResearchEnrollment, + ResearchEvent, + ResearchParticipant, + ResearchRecord, + ResearchRetentionJob, + TelemetryBatchReceipt, +) +from database.research_schemas import ( + ResearchSessionV1 as ResearchSessionRow, +) +from research.canonical import canonical_hash +from research.study.protocol.enums import RetentionAction + +from .identity import ( + deletion_ledger_record, + get_participant_by_account, + lock_participant_by_account, +) +from .models import DeletionLedgerEntry + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + +# Ledger ``reason`` distinguishing a participant's erasure request from a +# study's scheduled retention. +ERASURE_LEDGER_REASON = "participant_erasure" + + +@dataclass(frozen=True) +class ParticipantDataCounts: + enrollments: int = 0 + events: int = 0 + + +def _event_scope(enrollment_ids: list[uuid.UUID], session_ids: list[uuid.UUID]): + """Events belong to an enrollment directly or through one of its sessions.""" + condition = ResearchEvent.enrollment_id.in_(enrollment_ids) + if session_ids: + condition = or_(condition, ResearchEvent.research_session_id.in_(session_ids)) + return condition + + +def participant_data_counts(session: Session, account_id: uuid.UUID) -> ParticipantDataCounts: + """Count the research data held for an account (for the privacy summary).""" + participant = get_participant_by_account(session, account_id) + if participant is None: + return ParticipantDataCounts() + enrollment_ids = list( + session.execute( + select(ResearchEnrollment.enrollment_id).where( + ResearchEnrollment.participant_id == participant.participant_id + ) + ).scalars() + ) + if not enrollment_ids: + return ParticipantDataCounts() + session_ids = list( + session.execute( + select(ResearchSessionRow.session_id).where( + ResearchSessionRow.enrollment_id.in_(enrollment_ids) + ) + ).scalars() + ) + events = session.execute( + select(func.count()).select_from(ResearchEvent).where( + _event_scope(enrollment_ids, session_ids) + ) + ).scalar_one() + return ParticipantDataCounts(enrollments=len(enrollment_ids), events=int(events)) + + +def erase_participant_data( + session: Session, account_id: uuid.UUID, *, now: Optional[datetime] = None +) -> ParticipantDataCounts: + """Delete the account's research participation and telemetry; return counts.""" + # Pending ORM changes (e.g. a withdrawal in the same transaction) must reach + # the database before the bulk deletes remove their rows. + session.flush() + participant = lock_participant_by_account(session, account_id) + if participant is None: + return ParticipantDataCounts() + timestamp = now or datetime.now(timezone.utc) + enrollments = session.execute( + select(ResearchEnrollment.enrollment_id, ResearchEnrollment.study_id).where( + ResearchEnrollment.participant_id == participant.participant_id + ) + ).all() + erased_events = 0 + for enrollment_id, study_id in enrollments: + session_ids = list( + session.execute( + select(ResearchSessionRow.session_id).where( + ResearchSessionRow.enrollment_id == enrollment_id + ) + ).scalars() + ) + events = session.execute( + delete(ResearchEvent).where(_event_scope([enrollment_id], session_ids)) + ).rowcount + receipt_scope = TelemetryBatchReceipt.enrollment_id == enrollment_id + if session_ids: + receipt_scope = or_( + receipt_scope, TelemetryBatchReceipt.research_session_id.in_(session_ids) + ) + session.execute(delete(TelemetryBatchReceipt).where(receipt_scope)) + session.execute( + delete(ResearchRetentionJob).where( + ResearchRetentionJob.enrollment_id == enrollment_id + ) + ) + # Manual arm changes are audited per enrollment; they go with the data + # (the deletion ledger written below is a different record kind). + session.execute( + delete(ResearchRecord).where( + ResearchRecord.kind == RECORD_KIND_STUDY_LIFECYCLE, + ResearchRecord.scope_type == "enrollment", + ResearchRecord.scope_id == enrollment_id, + ) + ) + entry = DeletionLedgerEntry( + ledger_id=uuid.uuid4(), + enrollment_id=enrollment_id, + action=RetentionAction.DELETE_ALL, + applied_at=timestamp, + affected_count=events, + evidence_digest=canonical_hash( + { + "action": RetentionAction.DELETE_ALL.value, + "enrollment_id": str(enrollment_id), + "reason": ERASURE_LEDGER_REASON, + "remaining": [], + } + ), + ) + session.add( + deletion_ledger_record(entry, study_id=study_id, reason=ERASURE_LEDGER_REASON) + ) + erased_events += events + session.execute( + delete(ResearchParticipant).where( + ResearchParticipant.participant_id == participant.participant_id + ) + ) + return ParticipantDataCounts(enrollments=len(enrollments), events=erased_events) diff --git a/src/research/participants/identity.py b/src/research/participants/identity.py new file mode 100644 index 0000000..c8f593e --- /dev/null +++ b/src/research/participants/identity.py @@ -0,0 +1,1030 @@ +"""Participant identity and retention persistence. + +Pure enrollment and retention services plus the caller-supplied-session +persistence helpers and the researcher-safe projection. The invariants enforced +here are: + +* Telemetry is accepted only for an ``ACTIVE`` enrollment; every other state + (including unknown/absent) fails closed. +* Consent is a single acceptance recorded once at join (``consent_accepted_at``): + there is no document identity or re-consent. A participant withdraws by + opting the account out of data collection, which ends the enrollment as + ``WITHDRAWN``. +* One account has at most one ACTIVE enrollment across the platform. +* A study that ends marks its enrollments ``COMPLETED`` and bumps the revocation + epoch so previously issued capabilities stop working. +* Retention actions act on pseudonymous records and always emit a ledger entry. + +The persistence helpers take a caller-managed SQLAlchemy ``Session`` so the core +package never imports ``App`` or touches the application singleton. The router +is responsible for session lifecycle (``App.get_db_session`` / ``rollback`` / +``close``) and for access control over the private mapping. +""" + +from __future__ import annotations + +import secrets +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Iterable, Mapping, Optional, Sequence +from uuid import UUID + +from sqlalchemy import String, and_, case, cast, distinct, func, not_, or_, select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.exc import IntegrityError + +from database.db_schemas import Study as StudyRow +from database.research_schemas import ( + RECORD_KIND_RETENTION_EVIDENCE, + ResearchEnrollment, + ResearchEvent, + ResearchParticipant, + ResearchRecord, + ResearchRetentionJob, + StudyAssignment, +) +from database.research_schemas import ( + ResearchSessionV1 as ResearchSessionRow, +) +from database.db_schemas import Study as StudyRow +from research.budget import ledger as budget_ledger +from research.canonical import canonical_hash +from research.study.agents.enums import METERED_FRAMEWORKS +from research.runtime.sessions.enums import ( + CloseReason, + SessionReasonCode, + SessionState, +) +from research.runtime.sessions.models import SessionTransition +from research.study.protocol.enums import RetentionAction +from research.telemetry.enums import CanonicalEventType, EventSource + +from .enums import ( + EnrollmentStatus, + IdentityReasonCode, +) +from .models import ( + DeletionLedgerEntry, + Enrollment, + IdentityIssue, + Participant, + PseudonymousRecord, + ResearchEligibility, + RetentionResult, +) + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + +# Audit action used for the append-only deletion/retention ledger. +LEDGER_AUDIT_ACTION = "retention.deletion_ledger" + +# A duplicate enrollment request for the same revision returns the existing +# ACTIVE record rather than creating a second enrollment. +_REUSABLE_ENROLLMENT_STATES = frozenset({EnrollmentStatus.ACTIVE}) + +_ACTIVE_SESSION_STATES = ( + SessionState.NOT_STARTED.value, + SessionState.RUNNING.value, + SessionState.OFFLINE.value, + SessionState.SUSPENDED.value, +) + + +def _now(now: Optional[datetime] = None) -> datetime: + return now or datetime.now(timezone.utc) + + +def _issue(code: IdentityReasonCode, message: str, field: str = "") -> IdentityIssue: + return IdentityIssue(code=code, message=message, field=field) + + +def _isoformat(value: Optional[datetime]) -> Optional[str]: + return value.isoformat() if isinstance(value, datetime) else None + + +# --------------------------------------------------------------------------- +# Enrollment state machine +# --------------------------------------------------------------------------- + + +def generate_participant_code(*, byte_length: int = 12, prefix: str = "p") -> str: + """Return a random, opaque, study-local participant code. + + The code is drawn from a CSPRNG and is not derived from, nor reversible to, + the account identity. It carries no demographic or study semantics. + """ + return f"{prefix}_{secrets.token_hex(byte_length)}" + + +def default_eligibility(now: Optional[datetime] = None) -> ResearchEligibility: + """Return the permissive default eligibility result.""" + return ResearchEligibility( + eligible=True, + reasons=[IdentityReasonCode.ELIGIBLE], + evaluated_at=_now(now), + ) + + +def apply_retention( + enrollment: Enrollment, + records: Sequence[PseudonymousRecord], + action: RetentionAction, + *, + now: Optional[datetime] = None, +) -> RetentionResult: + """Apply a retention action to pseudonymous records and emit a ledger row. + + * ``RETAIN_ANONYMIZED`` keeps every record but strips account-linkable + fields. + * ``DELETE_IDENTIFIABLE`` removes records that still carry account-linkable + fields and keeps already-pseudonymous records. + * ``DELETE_ALL`` removes every record. + """ + projected: list[PseudonymousRecord] = [] + affected = 0 + + for record in records: + linkable = ( + record.account_id is not None + or record.email is not None + or record.session_token is not None + ) + + if action == RetentionAction.RETAIN_ANONYMIZED: + if linkable: + affected += 1 + projected.append( + record.model_copy( + update={"account_id": None, "email": None, "session_token": None} + ) + ) + elif action == RetentionAction.DELETE_IDENTIFIABLE: + if linkable: + affected += 1 + continue + projected.append(record) + elif action == RetentionAction.DELETE_ALL: + affected += 1 + continue + else: # pragma: no cover - enum is closed + continue + + ledger = DeletionLedgerEntry( + ledger_id=uuid.uuid4(), + enrollment_id=enrollment.enrollment_id, + action=action, + applied_at=_now(now), + affected_count=affected, + evidence_digest=canonical_hash( + { + "action": action.value, + "enrollment_id": str(enrollment.enrollment_id), + "remaining": [record.model_dump(mode="json") for record in projected], + } + ), + ) + return RetentionResult(action=action, records=projected, ledger=ledger) + + +# --------------------------------------------------------------------------- +# Researcher-safe projection +# --------------------------------------------------------------------------- + + +def researcher_projection(enrollment: Enrollment) -> dict[str, Any]: + """Return the non-identifying, researcher-visible view of an enrollment. + + Only study-local identifiers, status and timestamps are included. The + account id, participant id, email and session tokens are deliberately + absent. + """ + return { + "enrollment_id": str(enrollment.enrollment_id), + "participant_code": enrollment.participant_code, + "study_id": str(enrollment.study_id), + "status": enrollment.status.value, + "eligible": enrollment.eligibility.eligible, + "revocation_epoch": enrollment.revocation_epoch, + "retention_action": enrollment.retention_action.value, + "enrolled_at": _isoformat(enrollment.enrolled_at), + "updated_at": _isoformat(enrollment.updated_at), + } + + +# --------------------------------------------------------------------------- +# Participant self-view ("My studies") +# --------------------------------------------------------------------------- + +#: Participant-facing runtime names. A participant learns only the runtime +#: *kind* (whether an agent must be installed), never the arm it was randomized +#: to: no profile name/id, model or digest is ever derived from here. +RUNTIME_DISPLAY_NAMES: dict[str, str] = { + "code4me2-agent": "Code4Me agent (built-in)", + "goose": "Goose (install on your machine)", + "codex": "Codex (install on your machine)", +} + +#: A participant prompt is one ACP ``session/prompt`` observation. Streamed +#: assistant/thought chunks share the event type but carry ``message_kind`` or +#: the ``started`` lifecycle state (the study analytics apply the same rule). +PROMPT_EVENT_TYPE = CanonicalEventType.AGENT_MESSAGE_STARTED.value +#: A tool call is counted once across all of its ``tool.*`` lifecycle events. +TOOL_CALL_EVENT_TYPES = ( + CanonicalEventType.TOOL_CREATED.value, + CanonicalEventType.TOOL_STARTED.value, + CanonicalEventType.TOOL_COMPLETED.value, + CanonicalEventType.TOOL_FAILED.value, +) +_TERMINAL_SESSION_STATES = (SessionState.ENDED.value, SessionState.REVOKED.value) + + +def participant_runtime_view(snapshot: Any) -> Optional[dict[str, str]]: + """Runtime kind of a frozen assignment snapshot, or ``None``. + + Only ``framework_version`` is read from the snapshot; everything else in it + (profile identity, model, tools, digest) is arm detail the participant must + stay blind to. + """ + if not isinstance(snapshot, Mapping): + return None + framework = str(snapshot.get("framework_version") or "").strip().lower() + if not framework: + return None + return { + "framework_version": framework, + "display_name": RUNTIME_DISPLAY_NAMES.get(framework, framework), + # "shared": the study provides model access through its own key; + # "own": the participant signs in with their own account (Codex). + "credentials": "shared" if framework in METERED_FRAMEWORKS else "own", + } + + +def get_studies_by_id( + session: Session, study_ids: Iterable[uuid.UUID] +) -> dict[uuid.UUID, StudyRow]: + """Fetch study rows by id in one query (missing ids are simply absent).""" + ids = list({study_id for study_id in study_ids if study_id is not None}) + if not ids: + return {} + statement = select(StudyRow).where(StudyRow.study_id.in_(ids)) + return {row.study_id: row for row in session.execute(statement).scalars().all()} + + +def get_assignment_frameworks( + session: Session, enrollment_ids: Sequence[uuid.UUID] +) -> dict[uuid.UUID, Optional[dict[str, str]]]: + """``{enrollment_id: participant runtime view}`` from the frozen assignment. + + Reads only the snapshot's framework; enrollments without an assignment are + absent from the result. + """ + if not enrollment_ids: + return {} + statement = select( + StudyAssignment.enrollment_id, + StudyAssignment.profile_snapshot_json["framework_version"].astext, + ).where(StudyAssignment.enrollment_id.in_(list(enrollment_ids))) + return { + enrollment_id: participant_runtime_view({"framework_version": framework}) + for enrollment_id, framework in session.execute(statement).all() + } + + +def summarize_enrollment_sessions( + session: Session, enrollment_ids: Sequence[uuid.UUID] +) -> dict[uuid.UUID, dict[str, Any]]: + """``{enrollment_id: {total, active, last_activity_at}}`` in one query. + + ``active`` counts non-terminal sessions (neither ``ended`` nor ``revoked``), + the same rule as the study read metrics. + """ + if not enrollment_ids: + return {} + statement = ( + select( + ResearchSessionRow.enrollment_id, + func.count(ResearchSessionRow.session_id), + func.count(ResearchSessionRow.session_id).filter( + ResearchSessionRow.state.not_in(_TERMINAL_SESSION_STATES) + ), + func.max(ResearchSessionRow.last_activity_at), + ) + .where(ResearchSessionRow.enrollment_id.in_(list(enrollment_ids))) + .group_by(ResearchSessionRow.enrollment_id) + ) + return { + enrollment_id: { + "total": int(total or 0), + "active": int(active or 0), + "last_activity_at": _isoformat(last_activity_at), + } + for enrollment_id, total, active, last_activity_at in session.execute( + statement + ).all() + } + + +def summarize_enrollment_activity( + session: Session, enrollment_ids: Sequence[uuid.UUID] +) -> dict[uuid.UUID, dict[str, Any]]: + """``{enrollment_id: {prompts, tool_calls, last_event_at}}`` in two queries. + + Retention tombstones (``retention_state = 'DELETED'``) are excluded. The + counts follow the study analytics (``research.analysis.study_analytics`` + ``analyze_participant``) so a participant and their researcher see the + same numbers: + + * ``prompts`` counts ACP ``agent.message.started`` events that are not + streamed chunks (no ``payload.message_kind``, lifecycle not ``started``). + * ``tool_calls`` counts distinct calls over the ``tool.*`` events. The id is + ``correlations.tool_call_id``, else ``payload.tool_call_id`` (an empty + id is no id). An ACP call is one (emitter, id) of its session; an id-less + ACP event (such as ``terminal/create``) belongs to the call that owns it + and is not counted. The inference relay counts one call per id, or per + id-less event, and only in sessions without ACP calls: where the ACP + proxy observed the session, its calls are the authoritative ones. + """ + if not enrollment_ids: + return {} + ids = list(enrollment_ids) + envelope = ResearchEvent.envelope_json + retained = ResearchEvent.retention_state != "DELETED" + is_chunk = or_( + envelope[("payload", "message_kind")].astext.isnot(None), + func.coalesce(envelope["lifecycle_state"].astext, "") == "started", + ) + activity_statement = ( + select( + ResearchEvent.enrollment_id, + func.count(ResearchEvent.event_id).filter( + ResearchEvent.event_type == PROMPT_EVENT_TYPE, + ResearchEvent.source == EventSource.ACP.value, + not_(is_chunk), + ), + func.max(ResearchEvent.occurred_at), + ) + .where(ResearchEvent.enrollment_id.in_(ids), retained) + .group_by(ResearchEvent.enrollment_id) + ) + + is_relay = ResearchEvent.source == EventSource.RELAY.value + tool_call_id = func.nullif( + func.coalesce( + envelope[("correlations", "tool_call_id")].astext, + envelope[("payload", "tool_call_id")].astext, + ), + "", + ) + # JSON arrays keep the key parts unambiguous whatever the ids contain. + acp_key = case( + ( + and_(not_(is_relay), tool_call_id.isnot(None)), + cast(func.jsonb_build_array(ResearchEvent.emitter_id, tool_call_id), String), + ), + else_=None, + ) + relay_key = case( + ( + and_(is_relay, tool_call_id.isnot(None)), + cast(func.jsonb_build_array("id", tool_call_id), String), + ), + ( + is_relay, + cast( + func.jsonb_build_array( + "run", ResearchEvent.emitter_id, ResearchEvent.emitter_sequence + ), + String, + ), + ), + else_=None, + ) + per_session = ( + select( + ResearchEvent.enrollment_id.label("enrollment_id"), + func.count(distinct(acp_key)).label("acp_calls"), + func.count(distinct(relay_key)).label("relay_calls"), + ) + .where( + ResearchEvent.enrollment_id.in_(ids), + retained, + ResearchEvent.event_type.in_(TOOL_CALL_EVENT_TYPES), + ) + .group_by(ResearchEvent.enrollment_id, ResearchEvent.research_session_id) + .subquery() + ) + calls_statement = select( + per_session.c.enrollment_id, + func.sum( + case( + (per_session.c.acp_calls > 0, per_session.c.acp_calls), + else_=per_session.c.relay_calls, + ) + ), + ).group_by(per_session.c.enrollment_id) + tool_calls = { + enrollment_id: int(total or 0) + for enrollment_id, total in session.execute(calls_statement).all() + } + return { + enrollment_id: { + "prompts": int(prompts or 0), + "tool_calls": tool_calls.get(enrollment_id, 0), + "last_event_at": _isoformat(last_event_at), + } + for enrollment_id, prompts, last_event_at in session.execute( + activity_statement + ).all() + } + + +def participant_enrollment_details( + session: Session, enrollments: Sequence[Enrollment] +) -> dict[uuid.UUID, dict[str, Any]]: + """Batched per-enrollment facts for the participant's own study view. + + Five queries regardless of the number of enrollments. Each entry holds + ``study_row`` (the study row or ``None``), ``runtime`` (the runtime-kind + view or ``None``), ``sessions`` and ``activity`` summaries (zero-valued + when nothing was recorded). No arm detail is selected. + """ + enrollment_ids = [enrollment.enrollment_id for enrollment in enrollments] + studies = get_studies_by_id( + session, (enrollment.study_id for enrollment in enrollments) + ) + runtimes = get_assignment_frameworks(session, enrollment_ids) + sessions = summarize_enrollment_sessions(session, enrollment_ids) + activity = summarize_enrollment_activity(session, enrollment_ids) + budgets = budget_ledger.participant_views(session, enrollment_ids) + return { + enrollment.enrollment_id: { + "study_row": studies.get(enrollment.study_id), + "runtime": runtimes.get(enrollment.enrollment_id), + # Arm-blind budget numbers (unit, limit, consumed, remaining, flags). + "budget": budgets.get(enrollment.enrollment_id), + "sessions": sessions.get( + enrollment.enrollment_id, + {"total": 0, "active": 0, "last_activity_at": None}, + ), + "activity": activity.get( + enrollment.enrollment_id, + {"prompts": 0, "tool_calls": 0, "last_event_at": None}, + ), + } + for enrollment in enrollments + } + + +# --------------------------------------------------------------------------- +# Persistence helpers (caller-supplied session) +# --------------------------------------------------------------------------- + + +def _merge_job_evidence( + session: Session, + *, + enrollment_id: uuid.UUID, + action: RetentionAction, + key: str, + payload: dict[str, Any], +) -> None: + """Merge evidence onto the retention job for ``(enrollment, action)``. + + Policy/verification evidence belongs to the job so it survives even if the + enrollment row is later removed. + """ + statement = select(ResearchRetentionJob).where( + ResearchRetentionJob.enrollment_id == enrollment_id, + ResearchRetentionJob.action == action.value, + ) + job = session.execute(statement).scalars().first() + if not isinstance(job, ResearchRetentionJob): + return + evidence = job.evidence_json if isinstance(job.evidence_json, dict) else {} + merged = dict(evidence) + merged[key] = payload + job.evidence_json = merged + session.add(job) + + +def create_participant(session: Session, participant: Participant) -> ResearchParticipant: + """Insert the private account-to-participant mapping.""" + row = ResearchParticipant( + participant_id=participant.participant_id, + account_id=participant.account_id, + created_at=participant.created_at or _now(), + ) + session.add(row) + session.commit() + session.refresh(row) + return row + + +class ActiveEnrollmentConflict(Exception): + """Raised when the account-wide active-enrollment index would be violated. + + A typed conflict rather than a raw ``IntegrityError``, so the router can + report ``ALREADY_ENROLLED`` (409) naming the other enrollment instead of a + 500. + """ + + +def lock_participant_by_account( + session: Session, account_id: uuid.UUID +) -> Optional[ResearchParticipant]: + """Fetch the account's participant row with ``SELECT ... FOR UPDATE``. + + Taking this lock before the account-wide active-enrollment check is what + serializes two concurrent cross-study joins for the same account. + """ + statement = ( + select(ResearchParticipant) + .where(ResearchParticipant.account_id == account_id) + .with_for_update() + ) + return session.execute(statement).scalars().first() + + +def get_or_create_participant_row( + session: Session, + account_id: uuid.UUID, + *, + now: Optional[datetime] = None, + commit: bool = True, +) -> ResearchParticipant: + """Return the account's participant row, creating it race-safely and durably. + + Concurrent first-use uses ``INSERT ... ON CONFLICT DO NOTHING`` on the unique + ``account_id`` and then re-reads the winning row, so exactly one mapping + exists. The mapping is committed so it is durable and visible to the other + first-use transactions (a caller that needs the row lock re-acquires it with + :func:`lock_participant_by_account`). ``commit=False`` keeps the mapping in + the caller's transaction, which is required by web consent. + """ + row = session.execute( + select(ResearchParticipant).where( + ResearchParticipant.account_id == account_id + ) + ).scalars().first() + if row is None: + session.execute( + pg_insert(ResearchParticipant) + .values( + participant_id=uuid.uuid4(), + account_id=account_id, + created_at=_now(now), + ) + .on_conflict_do_nothing(index_elements=["account_id"]) + ) + if commit: + session.commit() + row = session.execute( + select(ResearchParticipant).where( + ResearchParticipant.account_id == account_id + ) + ).scalars().first() + if row is None: # pragma: no cover - the insert above guarantees the row + raise RuntimeError("participant mapping disappeared after an upsert") + return row + + +def get_active_enrollment_for_participant( + session: Session, + participant_id: uuid.UUID, + *, + exclude_enrollment_id: Optional[uuid.UUID] = None, +) -> Optional[ResearchEnrollment]: + """Return the account's live (active/pending/paused) enrollment, if any.""" + live_states = tuple(state.value for state in _REUSABLE_ENROLLMENT_STATES) + statement = select(ResearchEnrollment).where( + ResearchEnrollment.participant_id == participant_id, + ResearchEnrollment.status.in_(live_states), + ) + if exclude_enrollment_id is not None: + statement = statement.where( + ResearchEnrollment.enrollment_id != exclude_enrollment_id + ) + return session.execute(statement).scalars().first() + + +def create_enrollment(session: Session, enrollment: Enrollment) -> ResearchEnrollment: + """Insert an enrollment row, translating the one-active-index conflict. + + The partial unique index ``uq_research_enrollment_active_participant`` is the + database guard; a violation (e.g. a race that slipped past the participant + lock) is reported as a typed :class:`ActiveEnrollmentConflict`, never a 500. + """ + row = ResearchEnrollment( + enrollment_id=enrollment.enrollment_id, + participant_id=enrollment.participant_id, + study_id=enrollment.study_id, + participant_code=enrollment.participant_code, + status=enrollment.status.value, + revocation_epoch=enrollment.revocation_epoch, + eligibility_json=enrollment.eligibility.model_dump(mode="json"), + enrolled_at=enrollment.enrolled_at, + updated_at=enrollment.updated_at, + consent_accepted_at=enrollment.consent_accepted_at, + retention_action=enrollment.retention_action.value, + ) + try: + with session.begin_nested(): + session.add(row) + session.flush() + # The inference budget is born with the enrollment, in the same + # transaction, at the study's current default (0 = refuse until set). + study = session.get(StudyRow, enrollment.study_id) + budget_ledger.create_balance( + session, + enrollment_id=row.enrollment_id, + study_id=enrollment.study_id, + limit_micro_usd=int( + getattr(study, "inference_budget_default_micro_usd", 0) or 0 + ), + now=enrollment.enrolled_at, + ) + except IntegrityError as error: + raise ActiveEnrollmentConflict( + "this account already has an active enrollment" + ) from error + session.commit() + session.refresh(row) + return row + + +def get_participant( + session: Session, participant_id: uuid.UUID +) -> Optional[ResearchParticipant]: + """Fetch a participant mapping by id, or ``None``.""" + return session.get(ResearchParticipant, participant_id) + + +def get_participant_by_account( + session: Session, account_id: uuid.UUID +) -> Optional[ResearchParticipant]: + """Fetch the private mapping for a login account, or ``None``.""" + statement = select(ResearchParticipant).where( + ResearchParticipant.account_id == account_id + ) + return session.execute(statement).scalars().first() + + +def get_enrollment( + session: Session, enrollment_id: uuid.UUID, *, for_update: bool = False +) -> Optional[ResearchEnrollment]: + """Fetch an enrollment by id, or ``None``. + + ``for_update`` takes a row lock (``SELECT ... FOR UPDATE``) so a caller can + serialize with a concurrent transaction that touches the same enrollment. + """ + if not for_update: + return session.get(ResearchEnrollment, enrollment_id) + statement = ( + select(ResearchEnrollment) + .where(ResearchEnrollment.enrollment_id == enrollment_id) + .with_for_update() + ) + return session.execute(statement).scalars().first() + + +def get_enrollment_for_participant_study( + session: Session, participant_id: uuid.UUID, study_id: uuid.UUID +) -> Optional[ResearchEnrollment]: + """Fetch the participant's newest enrollment in one study, or ``None``. + + Used to refuse a self-service rejoin to a study the account already withdrew + from or completed, including via a different revision of that study. + """ + statement = ( + select(ResearchEnrollment) + .where( + ResearchEnrollment.participant_id == participant_id, + ResearchEnrollment.study_id == study_id, + ) + .order_by(ResearchEnrollment.enrolled_at.desc()) + ) + return session.execute(statement).scalars().first() + + +def list_enrollments( + session: Session, participant_id: uuid.UUID +) -> Sequence[ResearchEnrollment]: + """List a participant's enrollments, oldest-first.""" + statement = ( + select(ResearchEnrollment) + .where(ResearchEnrollment.participant_id == participant_id) + .order_by(ResearchEnrollment.enrolled_at.asc()) + ) + return list(session.execute(statement).scalars().all()) + + +def _apply_enrollment_fields( + row: ResearchEnrollment, enrollment: Enrollment +) -> None: + row.status = enrollment.status.value + row.revocation_epoch = enrollment.revocation_epoch + row.updated_at = enrollment.updated_at + row.retention_action = enrollment.retention_action.value + row.eligibility_json = enrollment.eligibility.model_dump(mode="json") + + +def update_enrollment( + session: Session, enrollment: Enrollment +) -> Optional[ResearchEnrollment]: + """Persist a status/epoch change onto an existing enrollment row.""" + row = session.get(ResearchEnrollment, enrollment.enrollment_id) + if row is None: + return None + _apply_enrollment_fields(row, enrollment) + session.commit() + session.refresh(row) + return row + + +def lock_enrollment( + session: Session, enrollment_id: uuid.UUID +) -> Optional[ResearchEnrollment]: + """Fetch an enrollment row with a ``SELECT ... FOR UPDATE`` row lock. + + The lock serializes concurrent enrollment transitions for one + enrollment; the caller owns the surrounding transaction and must commit (or + roll back) it. + """ + statement = ( + select(ResearchEnrollment) + .where(ResearchEnrollment.enrollment_id == enrollment_id) + .with_for_update() + ) + return session.execute(statement).scalars().first() + + +def revoke_active_sessions( + session: Session, + enrollment_id: uuid.UUID, + *, + now: datetime, + evidence_ref: str = "study_end", +) -> int: + """Mark every non-terminal session of an enrollment ``REVOKED``. + + Records a transition on each session and does not commit, so it can join the + caller's single transaction. + """ + statement = select(ResearchSessionRow).where( + ResearchSessionRow.enrollment_id == enrollment_id, + ResearchSessionRow.state.in_(_ACTIVE_SESSION_STATES), + ) + rows = list(session.execute(statement).scalars().all()) + for row in rows: + from_state = SessionState(row.state) + transition = SessionTransition( + transition_id=uuid.uuid4(), + research_session_id=row.session_id, + from_state=from_state, + to_state=SessionState.REVOKED, + occurred_at=now, + reason=SessionReasonCode.REVOKED, + close_reason=CloseReason.REVOKED, + evidence_ref=evidence_ref, + ) + existing = row.transitions_json + serialized = list(existing) if isinstance(existing, list) else [] + serialized.append(transition.model_dump(mode="json")) + row.transitions_json = serialized + row.state = SessionState.REVOKED.value + row.last_activity_at = now + row.closed_at = now + row.close_reason = CloseReason.REVOKED.value + session.add(row) + return len(rows) + + +def complete_enrollments_for_study( + session: Session, study_id: uuid.UUID, *, now: Optional[datetime] = None +) -> int: + """Mark a study's ACTIVE enrollments COMPLETED and revoke their sessions. + + This is the terminal transition for study end/expiry: the enrollment becomes + ``COMPLETED`` (terminal), its revocation epoch is bumped so + previously issued capabilities are rejected, and every non-terminal session is + revoked. Commits once. Idempotent: already-terminal enrollments are skipped. + """ + timestamp = _now(now) + statement = select(ResearchEnrollment).where( + ResearchEnrollment.study_id == study_id, + ResearchEnrollment.status == EnrollmentStatus.ACTIVE.value, + ).with_for_update() + rows = list(session.execute(statement).scalars().all()) + for row in rows: + revoke_active_sessions(session, row.enrollment_id, now=timestamp) + row.status = EnrollmentStatus.COMPLETED.value + row.updated_at = timestamp + session.add(row) + session.commit() + return len(rows) + + +def withdraw_active_enrollments( + session: Session, account_id: uuid.UUID, *, now: Optional[datetime] = None +) -> list[uuid.UUID]: + """End the account's ACTIVE enrollment as ``WITHDRAWN`` (participant opt-out). + + Mirrors the study-end transition: the revocation epoch is bumped so + previously issued capabilities are rejected, the sticky assignment is marked + withdrawn and every non-terminal session is revoked. Does not commit, so the + withdrawal joins the caller's transaction. Returns the withdrawn enrollment + ids (empty when nothing was active). + """ + participant = lock_participant_by_account(session, account_id) + if participant is None: + return [] + timestamp = _now(now) + statement = select(ResearchEnrollment).where( + ResearchEnrollment.participant_id == participant.participant_id, + ResearchEnrollment.status == EnrollmentStatus.ACTIVE.value, + ).with_for_update() + rows = list(session.execute(statement).scalars().all()) + for row in rows: + revoke_active_sessions( + session, + row.enrollment_id, + now=timestamp, + evidence_ref="participant_withdrawal", + ) + row.status = EnrollmentStatus.WITHDRAWN.value + row.revocation_epoch = int(row.revocation_epoch or 0) + 1 + row.updated_at = timestamp + session.add(row) + assignments = session.execute( + select(StudyAssignment).where( + StudyAssignment.enrollment_id == row.enrollment_id + ) + ).scalars().all() + for assignment in assignments: + assignment.status = EnrollmentStatus.WITHDRAWN.value + session.add(assignment) + return [row.enrollment_id for row in rows] + + +def _enqueue_retention_row( + session: Session, *, enrollment_id: uuid.UUID, action: RetentionAction, now: datetime +) -> Optional[ResearchRetentionJob]: + """Insert the PENDING retention job for ``(enrollment, action)`` if absent. + + Does not commit, so the enqueue joins the caller's transaction and the whole + workflow commits once. + """ + statement = select(ResearchRetentionJob).where( + ResearchRetentionJob.enrollment_id == enrollment_id, + ResearchRetentionJob.action == action.value, + ) + existing = session.execute(statement).scalars().first() + if existing is not None: + return existing + row = ResearchRetentionJob( + job_id=uuid.uuid4(), + enrollment_id=enrollment_id, + action=action.value, + state="PENDING", + attempts=0, + created_at=now, + affected_events=0, + evidence_json={}, + ) + session.add(row) + return row + + +def _retention_evidence_payload( + action: str, entry: DeletionLedgerEntry +) -> dict[str, Any]: + return { + "action": action, + "enrollment_id": str(entry.enrollment_id), + "record": entry.model_dump(mode="json"), + } + + +def deletion_ledger_record( + entry: DeletionLedgerEntry, + *, + study_id: Optional[uuid.UUID] = None, + reason: Optional[str] = None, +) -> ResearchRecord: + """Build the ``research_record`` row for one deletion ledger entry. + + The row is content-free: the enrollment id, counts, an evidence digest and + an optional machine-readable ``reason``. It never names the account. + """ + payload = _retention_evidence_payload(LEDGER_AUDIT_ACTION, entry) + if reason is not None: + payload["reason"] = reason + return ResearchRecord( + record_id=entry.ledger_id, + kind=RECORD_KIND_RETENTION_EVIDENCE, + scope_type="enrollment", + scope_id=entry.enrollment_id, + study_id=study_id, + actor=None, + occurred_at=entry.applied_at, + payload_json=payload, + ) + + +def insert_deletion_ledger( + session: Session, entry: DeletionLedgerEntry +) -> DeletionLedgerEntry: + """Append a deletion/retention ledger entry. + + The evidence is written to the generic ``research_record`` table + (``kind = RETENTION_EVIDENCE``) and, when a retention job exists for + ``(enrollment, action)``, merged into its ``evidence_json``. + """ + session.add(deletion_ledger_record(entry)) + _merge_job_evidence( + session, + enrollment_id=entry.enrollment_id, + action=entry.action, + key="ledger", + payload=entry.model_dump(mode="json"), + ) + session.commit() + return entry + + +def list_deletion_ledger( + session: Session, enrollment_id: Optional[uuid.UUID] = None +) -> Sequence[DeletionLedgerEntry]: + """List deletion ledger entries (optionally for one enrollment).""" + statement = select(ResearchRecord).where( + ResearchRecord.kind == RECORD_KIND_RETENTION_EVIDENCE, + ResearchRecord.payload_json["action"].astext == LEDGER_AUDIT_ACTION, + ) + if enrollment_id is not None: + statement = statement.where( + ResearchRecord.payload_json["enrollment_id"].astext == str(enrollment_id) + ) + statement = statement.order_by(ResearchRecord.occurred_at.asc()) + return [ + row_to_deletion_ledger(row) + for row in session.execute(statement).scalars().all() + ] + + +def row_to_participant(row: ResearchParticipant) -> Participant: + """Rehydrate the private participant mapping.""" + from .models import Participant as ParticipantModel + + return ParticipantModel( + participant_id=row.participant_id, + account_id=row.account_id, + created_at=row.created_at, + ) + + +def row_to_enrollment(row: ResearchEnrollment) -> Enrollment: + """Rehydrate an enrollment row into the domain model.""" + from .models import Enrollment as EnrollmentModel + + status = row.status + eligibility = ResearchEligibility.model_validate(row.eligibility_json or {}) + return EnrollmentModel( + enrollment_id=row.enrollment_id, + participant_id=row.participant_id, + study_id=row.study_id, + participant_code=row.participant_code, + status=status + if isinstance(status, EnrollmentStatus) + else EnrollmentStatus(status), + eligibility=eligibility, + enrolled_at=row.enrolled_at, + consent_accepted_at=row.consent_accepted_at, + revocation_epoch=row.revocation_epoch, + updated_at=row.updated_at, + retention_action=RetentionAction(row.retention_action), + ) + + +def row_to_deletion_ledger(row: ResearchRecord) -> DeletionLedgerEntry: + """Rehydrate a deletion ledger record row into the domain model.""" + from .models import DeletionLedgerEntry as DeletionLedgerEntryModel + + return DeletionLedgerEntryModel.model_validate(row.payload_json["record"]) + + +def deletion_ledger_summary(row: DeletionLedgerEntry) -> dict[str, Any]: + """Return a compact, non-secret deletion ledger summary.""" + applied_at = row.applied_at + action = row.action + return { + "ledger_id": str(row.ledger_id), + "enrollment_id": str(row.enrollment_id), + "action": action.value if isinstance(action, RetentionAction) else action, + "applied_at": ( + applied_at.isoformat() if isinstance(applied_at, datetime) else None + ), + "affected_count": row.affected_count, + "evidence_digest": row.evidence_digest, + } diff --git a/src/research/participants/models.py b/src/research/participants/models.py new file mode 100644 index 0000000..9776b9e --- /dev/null +++ b/src/research/participants/models.py @@ -0,0 +1,268 @@ +"""Pydantic v2 contracts for participant identity and retention. + +Consent is a single acceptance recorded once at join: there is no document +identity, no re-consent and no withdrawal. Privacy invariant: only +:class:`Participant` carries the private account-to-participant mapping. +Enrollment and every researcher projection are keyed by opaque UUIDs and a +random study-local ``participant_code`` that is never derived from, or parsed +back to, login identity. + +No retention model carries raw event content, an account id, an email or a +session token. A plan references events/exports by opaque id and a step +outcome; the evidence digest is computed over that plan plus counts and is +reproducible without reading content. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Any, Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import ( + EnrollmentStatus, + IdentityReasonCode, + RetentionAction, + RetentionJobState, + RetentionReasonCode, + RetentionStepOutcome, +) + +_BASE_CONFIG = ConfigDict(extra="forbid") +_FROZEN_CONFIG = ConfigDict(extra="forbid", frozen=True) + +_BASE = ConfigDict(extra="forbid") +_FROZEN = ConfigDict(extra="forbid", frozen=True) + +__all__ = [ + "DeletionLedgerEntry", + "Enrollment", + "EnrollmentResult", + "IdentityIssue", + "Participant", + "PseudonymousRecord", + "ResearchEligibility", + "RetentionEventOutcome", + "RetentionEvidence", + "RetentionExecution", + "RetentionJob", + "RetentionPlan", + "RetentionResult", +] + + +# --------------------------------------------------------------------------- +# Identity +# --------------------------------------------------------------------------- + + +class Participant(BaseModel): + """Private account-to-participant mapping. + + This is the only object that links a login account to a study participant. + It is privileged: it is never returned to researchers and is never embedded + in a researcher projection or export. + """ + + model_config = _BASE_CONFIG + + participant_id: UUID + account_id: UUID + created_at: datetime + + +class ResearchEligibility(BaseModel): + """Typed eligibility result, evaluated before enrollment.""" + + model_config = _BASE_CONFIG + + eligible: bool + reasons: list[IdentityReasonCode] = Field(default_factory=list) + evaluated_at: Optional[datetime] = None + + +class Enrollment(BaseModel): + """One participant's study enrollment and consent identity.""" + + model_config = _BASE_CONFIG + + enrollment_id: UUID + participant_id: UUID + study_id: UUID + # Random, opaque, study-local. Never derived from the account id and never + # parsed for meaning; it is the only participant handle researchers see. + participant_code: str + status: EnrollmentStatus = EnrollmentStatus.ACTIVE + eligibility: ResearchEligibility + enrolled_at: datetime + # Set at join: the participant accepted the global consent text once. + consent_accepted_at: Optional[datetime] = None + revocation_epoch: int = 0 + updated_at: datetime + retention_action: RetentionAction = RetentionAction.RETAIN_ANONYMIZED + + +class DeletionLedgerEntry(BaseModel): + """Append-only evidence that a retention action was applied.""" + + model_config = _BASE_CONFIG + + ledger_id: UUID + enrollment_id: UUID + action: RetentionAction + applied_at: datetime + affected_count: int + evidence_digest: str + + +class PseudonymousRecord(BaseModel): + """A research record that may carry account-linkable fields. + + ``account_id``, ``email`` and ``session_token`` are the linkable fields a + retention action strips or removes. ``participant_code`` and + ``pseudonymous_payload`` are the safe research fields. + """ + + model_config = _BASE_CONFIG + + record_id: str + enrollment_id: Optional[UUID] = None + participant_code: Optional[str] = None + account_id: Optional[UUID] = None + email: Optional[str] = None + session_token: Optional[str] = None + pseudonymous_payload: dict[str, Any] = Field(default_factory=dict) + + +class IdentityIssue(BaseModel): + """One typed identity rejection reason.""" + + model_config = _BASE_CONFIG + + code: IdentityReasonCode + message: str + field: str = "" + + +class EnrollmentResult(BaseModel): + """Typed outcome of an enrollment operation.""" + + model_config = _BASE_CONFIG + + accepted: bool + enrollment: Optional[Enrollment] = None + created: bool = False + reused: bool = False + issue: Optional[IdentityIssue] = None + + +class RetentionResult(BaseModel): + """Typed outcome of applying a retention action to pseudonymous records.""" + + model_config = _BASE_CONFIG + + action: RetentionAction + records: list[PseudonymousRecord] = Field(default_factory=list) + ledger: DeletionLedgerEntry + + +# --------------------------------------------------------------------------- +# Retention execution +# --------------------------------------------------------------------------- + + +class RetentionEventOutcome(BaseModel): + """The planned outcome for one stored event (id + step only).""" + + model_config = _FROZEN + + event_id: UUID + outcome: RetentionStepOutcome + + +class RetentionPlan(BaseModel): + """A deterministic plan for one retention action over one enrollment.""" + + model_config = _FROZEN + + enrollment_id: UUID + action: RetentionAction + event_outcomes: list[RetentionEventOutcome] = Field(default_factory=list) + plan_digest: str + + def _event_ids(self, outcome: RetentionStepOutcome) -> list[UUID]: + return [ + item.event_id + for item in self.event_outcomes + if item.outcome == outcome + ] + + def deleted_event_ids(self) -> list[UUID]: + """Event ids that will be deleted/tombstoned.""" + return self._event_ids(RetentionStepOutcome.DELETED) + + def anonymized_event_ids(self) -> list[UUID]: + """Event ids that will be kept but anonymized.""" + return self._event_ids(RetentionStepOutcome.ANONYMIZED) + + def affected_event_ids(self) -> list[UUID]: + """Event ids whose stored state changes (deleted or anonymized).""" + return self.deleted_event_ids() + self.anonymized_event_ids() + + @property + def affected_events(self) -> int: + """Number of events whose stored state changes.""" + return len(self.affected_event_ids()) + + +class RetentionJob(BaseModel): + """Durable, idempotent state machine for one enrollment/action.""" + + model_config = _BASE + + job_id: UUID + enrollment_id: UUID + action: RetentionAction + state: RetentionJobState = RetentionJobState.PENDING + attempts: int = 0 + created_at: datetime + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + last_error: Optional[str] = None + evidence_digest: Optional[str] = None + affected_events: int = 0 + reason: RetentionReasonCode = RetentionReasonCode.OK + + +class RetentionEvidence(BaseModel): + """Immutable, content-free evidence that a retention action ran. + + ``ledger_id`` is persisted as a ``research_record`` row + (``kind = RETENTION_EVIDENCE``) and merged into the matching + ``research_retention_job.evidence_json`` so an operator can verify execution + without reading any event content. + """ + + model_config = _FROZEN + + ledger_id: UUID + enrollment_id: UUID + action: RetentionAction + applied_at: datetime + affected_events: int + evidence_digest: str + + +class RetentionExecution(BaseModel): + """Result of executing (or re-executing) one retention job.""" + + model_config = _BASE + + job: RetentionJob + plan: Optional[RetentionPlan] = None + evidence: Optional[RetentionEvidence] = None + deleted_event_ids: list[UUID] = Field(default_factory=list) + anonymized_event_ids: list[UUID] = Field(default_factory=list) + reused: bool = False diff --git a/src/research/participants/retention.py b/src/research/participants/retention.py new file mode 100644 index 0000000..8ca00a9 --- /dev/null +++ b/src/research/participants/retention.py @@ -0,0 +1,923 @@ +"""Pure retention planning/execution, persistence helpers and worker entry points. + +This module is deterministic and framework-free: it never imports ``App``, +FastAPI or the ORM eagerly. Given a job and the enrollment's stored events it +produces a plan, applies the published retention action (via +:func:`apply_retention` semantics) through an +injected applier, and transitions the job state. + +The SQLAlchemy helpers take a caller-supplied ``Session`` (like every other +research store) and return ORM rows; :class:`SqlAlchemyRetentionStore` is a +thin, injectable wrapper that returns domain objects to the worker. + +The worker processes a bounded batch with retries: a job becomes ``RUNNING``, +executes, and finishes ``COMPLETED`` or ``RETRYABLE``/``FAILED``. Exceptions +never crash the process; they are recorded on the job with a content-free +evidence digest. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Iterable, Optional, Protocol, Sequence +from uuid import UUID + +from sqlalchemy import select + +from database.research_schemas import ( + ResearchAgentRun, + ResearchEnrollment, + ResearchEvent, + ResearchRetentionJob, +) +from database.research_schemas import ( + ResearchSessionV1 as ResearchSessionRow, +) +from research.canonical import canonical_hash +from research.runtime.sessions.enums import ( + AgentRunOutcome, + CloseReason, + SessionState, +) +from research.study.protocol.enums import RetentionAction +from research.telemetry.ingestion.store import row_to_record + +from .enums import ( + RetentionJobState, + RetentionReasonCode, + RetentionState, + RetentionStepOutcome, +) +from .identity import PseudonymousRecord, apply_retention, insert_deletion_ledger +from .models import ( + DeletionLedgerEntry, + RetentionEventOutcome, + RetentionEvidence, + RetentionExecution, + RetentionJob, + RetentionPlan, +) + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + from research.telemetry.ingestion.models import ResearchEventRecord + + from .models import Enrollment + +__all__ = [ + "RetentionApplier", + "SqlAlchemyRetentionStore", + "apply_event_anonymization", + "apply_event_deletion", + "enqueue_for_enrollment", + "event_to_pseudonymous_record", + "execute_retention", + "get_enrollment_study_id", + "get_retention_job", + "get_retention_job_for_enrollment_action", + "is_event_visible", + "job_summary", + "list_candidate_events", + "list_retention_jobs_for_enrollment", + "plan_retention", + "revoke_sessions", + "row_to_retention_job", + "run_retention_for_enrollment", + "run_retention_job", + "start_retention_job", + "upsert_retention_job", +] + +_LINKABLE_KEYS = ("account_id", "email", "session_token") + + +class RetentionApplier(Protocol): + """Side-effecting steps the pure executor calls (implemented by the store).""" + + def apply_event_retention( + self, + deleted_event_ids: Sequence[UUID], + anonymized_event_ids: Sequence[UUID], + *, + now: datetime, + ) -> None: # pragma: no cover - structural protocol + ... + + def apply_session_retention( + self, + enrollment_id: UUID, + *, + now: datetime, + ) -> None: # pragma: no cover - structural protocol + ... + + +class _NullApplier: + """Default in-memory applier: plans but persists nothing.""" + + def apply_event_retention( + self, + deleted_event_ids: Sequence[UUID], + anonymized_event_ids: Sequence[UUID], + *, + now: datetime, + ) -> None: + return None + + def apply_session_retention( + self, + enrollment_id: UUID, + *, + now: datetime, + ) -> None: + return None + + +class _EnrollmentRef: + """Minimal object carrying the one attribute ``apply_retention`` reads.""" + + __slots__ = ("enrollment_id",) + + def __init__(self, enrollment_id: UUID) -> None: + self.enrollment_id = enrollment_id + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def _coerce_uuid(value: Any) -> Optional[UUID]: + if value is None: + return None + if isinstance(value, UUID): + return value + try: + return UUID(str(value)) + except (TypeError, ValueError, AttributeError): + return None + + +def _envelope(event: Any) -> dict[str, Any]: + envelope = getattr(event, "envelope", None) + return dict(envelope) if isinstance(envelope, dict) else {} + + +def _payload(event: Any) -> dict[str, Any]: + # Stored records carry content only inside their canonical envelope; a + # canonical event exposes ``payload`` directly. + payload = _envelope(event).get("payload") + if not isinstance(payload, dict): + payload = getattr(event, "payload", None) + return dict(payload) if isinstance(payload, dict) else {} + + +def _provenance(event: Any) -> dict[str, Any]: + provenance = _envelope(event).get("provenance") + if not isinstance(provenance, dict): + provenance = getattr(event, "provenance", None) + return dict(provenance) if isinstance(provenance, dict) else {} + + +def _extract_linkable(event: Any) -> dict[str, Any]: + """Return the present account-linkable fields from payload/provenance. + + Only the presence of a value matters for linkability; the values are never + logged or persisted by this module. + """ + payload = _payload(event) + provenance = _provenance(event) + found: dict[str, Any] = {} + for key in _LINKABLE_KEYS: + value = payload.get(key) + if value is None: + value = provenance.get(key) + if value is not None: + found[key] = value + return found + + +def is_event_visible(event: Any) -> bool: + """Whether a stored event is still part of read models and exports. + + A ``DELETED`` tombstone is never visible; ``ANONYMIZED`` and ``RETAINED`` + events remain exportable. + """ + state = getattr(event, "retention_state", RetentionState.RETAINED.value) + return state != RetentionState.DELETED.value + + +def event_to_pseudonymous_record(event: Any) -> PseudonymousRecord: + """Project one stored event into the identity retention vocabulary. + + Only ``account_id``, ``email`` and ``session_token`` count as linkable, so + a normal pseudonymous event is a no-op for ``DELETE_IDENTIFIABLE`` while a + planted linkable payload is removed or stripped exactly as + :func:`apply_retention` specifies. + """ + payload = _payload(event) + provenance = _provenance(event) + account_raw = payload.get("account_id") + if account_raw is None: + account_raw = provenance.get("account_id") + email_raw = payload.get("email") + if email_raw is None: + email_raw = provenance.get("email") + token_raw = payload.get("session_token") + if token_raw is None: + token_raw = provenance.get("session_token") + + pseudonymous_payload = { + key: value + for key, value in payload.items() + if key not in _LINKABLE_KEYS + } + return PseudonymousRecord( + record_id=str(event.event_id), + enrollment_id=getattr(event, "enrollment_id", None), + participant_code=None, + account_id=_coerce_uuid(account_raw), + email=str(email_raw) if email_raw is not None else None, + session_token=str(token_raw) if token_raw is not None else None, + pseudonymous_payload=pseudonymous_payload, + ) + + +def _plan_digest( + enrollment_id: UUID, + action: RetentionAction, + event_outcomes: Sequence[RetentionEventOutcome], +) -> str: + return canonical_hash( + { + "action": action.value, + "enrollment_id": str(enrollment_id), + "events": sorted( + (str(item.event_id), item.outcome.value) for item in event_outcomes + ), + } + ) + + +def _evidence_digest(plan: RetentionPlan) -> str: + return canonical_hash( + { + "plan_digest": plan.plan_digest, + "action": plan.action.value, + "enrollment_id": str(plan.enrollment_id), + "affected_events": plan.affected_events, + } + ) + + +def plan_retention( + enrollment: Enrollment, + action: RetentionAction, + records: Iterable[ResearchEventRecord], + *, + now: Optional[datetime] = None, +) -> RetentionPlan: + """Return a deterministic plan for one enrollment and retention action. + + The event outcomes reuse :func:`apply_retention` + so ``DELETE_ALL``, ``DELETE_IDENTIFIABLE`` and ``RETAIN_ANONYMIZED`` mean + exactly what they mean for pseudonymous records. + """ + timestamp = _now(now) + events = list(records) + pseudonyms = [event_to_pseudonymous_record(event) for event in events] + result = apply_retention(enrollment, pseudonyms, action, now=timestamp) + retained_ids = {record.record_id for record in result.records} + + event_outcomes: list[RetentionEventOutcome] = [] + for event in events: + record_id = str(event.event_id) + if record_id not in retained_ids: + outcome = RetentionStepOutcome.DELETED + elif action == RetentionAction.RETAIN_ANONYMIZED and _extract_linkable(event): + outcome = RetentionStepOutcome.ANONYMIZED + else: + outcome = RetentionStepOutcome.RETAINED + event_outcomes.append( + RetentionEventOutcome(event_id=event.event_id, outcome=outcome) + ) + + return RetentionPlan( + enrollment_id=enrollment.enrollment_id, + action=action, + event_outcomes=event_outcomes, + plan_digest=_plan_digest(enrollment.enrollment_id, action, event_outcomes), + ) + + +def start_retention_job( + job: RetentionJob, *, now: Optional[datetime] = None +) -> RetentionJob: + """Transition a job to ``RUNNING`` and increment its attempt counter.""" + timestamp = _now(now) + return job.model_copy( + update={ + "state": RetentionJobState.RUNNING, + "attempts": job.attempts + 1, + "started_at": job.started_at or timestamp, + "completed_at": None, + } + ) + + +def _build_evidence( + job: RetentionJob, plan: RetentionPlan, timestamp: datetime +) -> RetentionEvidence: + return RetentionEvidence( + ledger_id=uuid.uuid4(), + enrollment_id=job.enrollment_id, + action=job.action, + applied_at=timestamp, + affected_events=plan.affected_events, + evidence_digest=_evidence_digest(plan), + ) + + +def _evidence_from_job(job: RetentionJob, timestamp: datetime) -> RetentionEvidence: + return RetentionEvidence( + ledger_id=uuid.uuid4(), + enrollment_id=job.enrollment_id, + action=job.action, + applied_at=job.completed_at or timestamp, + affected_events=job.affected_events, + evidence_digest=job.evidence_digest or "", + ) + + +def execute_retention( + job: RetentionJob, + events: Iterable[ResearchEventRecord], + action: RetentionAction, + now: Optional[datetime] = None, + *, + applier: Optional[RetentionApplier] = None, + max_attempts: int = 3, +) -> RetentionExecution: + """Execute (or re-execute) one retention job. + + * A ``COMPLETED`` job is a no-op returning the same job and evidence digest. + * A ``FAILED``/``RETRYABLE``/``PENDING`` job is planned and executed; the + attempt counter starts at the current value. + * On an applier failure the job becomes ``RETRYABLE`` while attempts remain + and ``FAILED`` afterwards; it is never ``COMPLETED`` and the plan/evidence + is returned so the caller can retain the ledger. + """ + timestamp = _now(now) + + if job.state == RetentionJobState.COMPLETED: + return RetentionExecution( + job=job, reused=True, evidence=_evidence_from_job(job, timestamp) + ) + + if action != job.action: + failed = job.model_copy( + update={ + "state": RetentionJobState.FAILED, + "started_at": job.started_at or timestamp, + "completed_at": timestamp, + "last_error": RetentionReasonCode.ACTION_MISMATCH.value, + "reason": RetentionReasonCode.ACTION_MISMATCH, + } + ) + return RetentionExecution(job=failed, reused=False) + + running = ( + job + if job.state == RetentionJobState.RUNNING + else start_retention_job(job, now=timestamp) + ) + plan = plan_retention( + _EnrollmentRef(running.enrollment_id), + action, + list(events), + now=timestamp, + ) + evidence = _build_evidence(running, plan, timestamp) + deleted = plan.deleted_event_ids() + anonymized = plan.anonymized_event_ids() + active_applier: RetentionApplier = applier or _NullApplier() + + try: + # DELETE_ALL also revokes the enrollment's sessions/runs, so a later + # re-export that reads sessions independently of events cannot resurrect + # the deleted participant's activity. + if action == RetentionAction.DELETE_ALL: + active_applier.apply_session_retention( + running.enrollment_id, now=timestamp + ) + if deleted or anonymized: + active_applier.apply_event_retention(deleted, anonymized, now=timestamp) + except Exception as error: # noqa: BLE001 - any step failure is a job failure + retryable = running.attempts < max_attempts + state = ( + RetentionJobState.RETRYABLE if retryable else RetentionJobState.FAILED + ) + reason = ( + RetentionReasonCode.PARTIAL_FAILURE + if retryable + else RetentionReasonCode.RETRY_EXHAUSTED + ) + failed_job = running.model_copy( + update={ + "state": state, + "completed_at": None, + "last_error": type(error).__name__, + "evidence_digest": evidence.evidence_digest, + "affected_events": plan.affected_events, + "reason": reason, + } + ) + return RetentionExecution( + job=failed_job, + plan=plan, + evidence=evidence, + deleted_event_ids=deleted, + anonymized_event_ids=anonymized, + reused=False, + ) + + completed = running.model_copy( + update={ + "state": RetentionJobState.COMPLETED, + "completed_at": timestamp, + "last_error": None, + "evidence_digest": evidence.evidence_digest, + "affected_events": plan.affected_events, + "reason": RetentionReasonCode.OK, + } + ) + return RetentionExecution( + job=completed, + plan=plan, + evidence=evidence, + deleted_event_ids=deleted, + anonymized_event_ids=anonymized, + reused=False, + ) + + +# --------------------------------------------------------------------------- +# SQLAlchemy helpers (session supplied by the caller) +# --------------------------------------------------------------------------- + + +def get_enrollment_study_id( + session: Session, enrollment_id: UUID +) -> Optional[UUID]: + """Return the study a stored enrollment belongs to, or ``None``.""" + row = session.get(ResearchEnrollment, enrollment_id) + return row.study_id if row is not None else None + + +def list_candidate_events( + session: Session, enrollment_id: UUID +) -> list[ResearchEventRecord]: + """Load an enrollment's non-tombstoned events as domain records.""" + statement = ( + select(ResearchEvent) + .where( + ResearchEvent.enrollment_id == enrollment_id, + ResearchEvent.retention_state != RetentionState.DELETED.value, + ) + .order_by(ResearchEvent.occurred_at.asc(), ResearchEvent.event_id.asc()) + ) + return [ + row_to_record(row) + for row in session.execute(statement).scalars().all() + ] + + +def _clear_event_content(row: ResearchEvent, now: datetime) -> None: + row.retention_state = RetentionState.DELETED.value + row.anonymized_at = now + row.enrollment_id = None + row.research_session_id = None + row.agent_run_id = None + # The complete envelope is content-free too: no copy of the deleted content + # may survive in any column or in the envelope. + row.envelope_json = {} + + +def apply_event_deletion( + session: Session, event_ids: Sequence[UUID], now: datetime +) -> int: + """Tombstone events: clear all content and mark them deleted.""" + ids = list(event_ids) + if not ids: + return 0 + statement = select(ResearchEvent).where(ResearchEvent.event_id.in_(ids)) + rows = list(session.execute(statement).scalars().all()) + for row in rows: + _clear_event_content(row, now) + session.commit() + return len(rows) + + +def _strip_linkable(mapping: Any) -> dict[str, Any]: + if not isinstance(mapping, dict): + return {} + return {key: value for key, value in mapping.items() if key not in _LINKABLE_KEYS} + + +def apply_event_anonymization( + session: Session, event_ids: Sequence[UUID], now: datetime +) -> int: + """Strip account-linkable fields and mark events anonymized.""" + ids = list(event_ids) + if not ids: + return 0 + statement = select(ResearchEvent).where(ResearchEvent.event_id.in_(ids)) + rows = list(session.execute(statement).scalars().all()) + for row in rows: + row.retention_state = RetentionState.ANONYMIZED.value + row.anonymized_at = now + # The envelope is the only content authority, so it is scrubbed in place. + envelope = dict(getattr(row, "envelope_json", None) or {}) + for key in ("payload", "provenance", "correlations", "unknown_payload"): + if key in envelope: + if isinstance(envelope[key], dict): + envelope[key] = _strip_linkable(envelope[key]) + else: + envelope.pop(key, None) + row.envelope_json = envelope + session.commit() + return len(rows) + + +def revoke_sessions(session: Session, enrollment_id: UUID, now: datetime) -> int: + """Mark an enrollment's sessions (and their runs) revoked for DELETE_ALL. + + A tombstoned enrollment must not leave live session/run rows behind: a + later re-export reads sessions independently of events, so without this a + deleted participant's activity could reappear in a fresh bundle. + """ + statement = select(ResearchSessionRow).where( + ResearchSessionRow.enrollment_id == enrollment_id + ) + sessions = list(session.execute(statement).scalars().all()) + if not sessions: + return 0 + session_ids = [row.session_id for row in sessions] + for row in sessions: + row.state = SessionState.REVOKED.value + row.closed_at = now + row.close_reason = CloseReason.REVOKED.value + run_statement = select(ResearchAgentRun).where( + ResearchAgentRun.research_session_id.in_(session_ids) + ) + for run in session.execute(run_statement).scalars().all(): + if run.ended_at is None: + run.ended_at = now + run.outcome = AgentRunOutcome.REVOKED.value + session.commit() + return len(sessions) + + +def _apply_job_fields(row: ResearchRetentionJob, job: RetentionJob) -> None: + row.enrollment_id = job.enrollment_id + row.action = job.action.value + row.state = job.state.value + row.attempts = job.attempts + row.created_at = job.created_at + row.started_at = job.started_at + row.completed_at = job.completed_at + row.last_error = job.last_error + row.evidence_digest = job.evidence_digest + row.affected_events = job.affected_events + + +def upsert_retention_job( + session: Session, job: RetentionJob +) -> ResearchRetentionJob: + """Insert or update one retention job (idempotent per enrollment/action).""" + row = session.get(ResearchRetentionJob, job.job_id) + if row is None: + existing = get_retention_job_for_enrollment_action( + session, job.enrollment_id, job.action + ) + if existing is not None: + return existing + row = ResearchRetentionJob(job_id=job.job_id) + _apply_job_fields(row, job) + session.add(row) + else: + _apply_job_fields(row, job) + session.commit() + session.refresh(row) + return row + + +def get_retention_job( + session: Session, job_id: UUID +) -> Optional[ResearchRetentionJob]: + """Fetch a retention job row by id, or ``None``.""" + return session.get(ResearchRetentionJob, job_id) + + +def get_retention_job_for_enrollment_action( + session: Session, enrollment_id: UUID, action: RetentionAction +) -> Optional[ResearchRetentionJob]: + """Fetch the unique job for ``(enrollment, action)``, or ``None``.""" + statement = select(ResearchRetentionJob).where( + ResearchRetentionJob.enrollment_id == enrollment_id, + ResearchRetentionJob.action == action.value, + ) + return session.execute(statement).scalars().first() + + +def list_retention_jobs_for_enrollment( + session: Session, enrollment_id: UUID +) -> list[ResearchRetentionJob]: + """List an enrollment's retention jobs, oldest-first.""" + statement = ( + select(ResearchRetentionJob) + .where(ResearchRetentionJob.enrollment_id == enrollment_id) + .order_by(ResearchRetentionJob.created_at.asc()) + ) + return list(session.execute(statement).scalars().all()) + + +def row_to_retention_job(row: ResearchRetentionJob) -> RetentionJob: + """Rehydrate a retention job row into the domain model.""" + state = RetentionJobState(row.state) + if state == RetentionJobState.RETRYABLE: + reason = RetentionReasonCode.PARTIAL_FAILURE + elif state == RetentionJobState.FAILED: + reason = RetentionReasonCode.RETRY_EXHAUSTED + else: + reason = RetentionReasonCode.OK + return RetentionJob( + job_id=row.job_id, + enrollment_id=row.enrollment_id, + action=RetentionAction(row.action), + state=state, + attempts=row.attempts or 0, + created_at=row.created_at, + started_at=row.started_at, + completed_at=row.completed_at, + last_error=row.last_error, + evidence_digest=row.evidence_digest, + affected_events=row.affected_events or 0, + reason=reason, + ) + + +def job_summary(job: RetentionJob) -> dict[str, Any]: + """Compact, non-secret job summary safe for operator responses.""" + return { + "job_id": str(job.job_id), + "enrollment_id": str(job.enrollment_id), + "action": job.action.value, + "state": job.state.value, + "attempts": job.attempts, + "created_at": _iso(job.created_at), + "started_at": _iso(job.started_at), + "completed_at": _iso(job.completed_at), + "last_error": job.last_error, + "evidence_digest": job.evidence_digest, + "affected_events": job.affected_events, + "reason": job.reason.value, + } + + +def _iso(value: Any) -> Optional[str]: + return value.isoformat() if isinstance(value, datetime) else None + + +def insert_retention_evidence( + session: Session, evidence: RetentionEvidence +) -> Any: + """Append a content-free deletion-ledger row for an executed job.""" + entry = DeletionLedgerEntry( + ledger_id=evidence.ledger_id, + enrollment_id=evidence.enrollment_id, + action=evidence.action, + applied_at=evidence.applied_at, + affected_count=evidence.affected_events, + evidence_digest=evidence.evidence_digest, + ) + return insert_deletion_ledger(session, entry) + + +# --------------------------------------------------------------------------- +# Injectable SQLAlchemy wrapper +# --------------------------------------------------------------------------- + + +class SqlAlchemyRetentionStore: + """Adapter exposing the retention persistence surface over one session.""" + + def __init__(self, session: Session) -> None: + self.session = session + + def get_enrollment_study_id(self, enrollment_id: UUID) -> Optional[UUID]: + return get_enrollment_study_id(self.session, enrollment_id) + + def load_events(self, enrollment_id: UUID) -> list[ResearchEventRecord]: + return list_candidate_events(self.session, enrollment_id) + + def mark_events_deleted(self, event_ids: Sequence[UUID], now: datetime) -> int: + return apply_event_deletion(self.session, event_ids, now) + + def anonymize_events(self, event_ids: Sequence[UUID], now: datetime) -> int: + return apply_event_anonymization(self.session, event_ids, now) + + def revoke_sessions(self, enrollment_id: UUID, now: datetime) -> int: + return revoke_sessions(self.session, enrollment_id, now) + + def get_job(self, job_id: UUID) -> Optional[RetentionJob]: + row = get_retention_job(self.session, job_id) + return row_to_retention_job(row) if row is not None else None + + def get_job_for_enrollment_action( + self, enrollment_id: UUID, action: RetentionAction + ) -> Optional[RetentionJob]: + row = get_retention_job_for_enrollment_action( + self.session, enrollment_id, action + ) + return row_to_retention_job(row) if row is not None else None + + def upsert_job(self, job: RetentionJob) -> RetentionJob: + return row_to_retention_job(upsert_retention_job(self.session, job)) + + def insert_evidence(self, evidence: RetentionEvidence) -> None: + insert_retention_evidence(self.session, evidence) + + +# --------------------------------------------------------------------------- +# Worker entry points +# --------------------------------------------------------------------------- + + +class _StoreApplier: + """Applies a plan through the retention persistence surface.""" + + def __init__(self, api: Any) -> None: + self._api = api + + def apply_event_retention( + self, + deleted_event_ids: Sequence[UUID], + anonymized_event_ids: Sequence[UUID], + *, + now: datetime, + ) -> None: + if deleted_event_ids: + self._api.mark_events_deleted(deleted_event_ids, now) + if anonymized_event_ids: + self._api.anonymize_events(anonymized_event_ids, now) + + def apply_session_retention(self, enrollment_id: UUID, *, now: datetime) -> None: + self._api.revoke_sessions(enrollment_id, now) + + +def _resolve_store(session: Any, store_api: Optional[Any]) -> Any: + if store_api is not None: + return store_api + if session is None: + raise ValueError("a session or an explicit retention store is required") + return SqlAlchemyRetentionStore(session) + + +def _failure_evidence( + job: RetentionJob, error: Exception, timestamp: datetime +) -> RetentionEvidence: + digest = canonical_hash( + { + "action": job.action.value, + "enrollment_id": str(job.enrollment_id), + "error": type(error).__name__, + } + ) + return RetentionEvidence( + ledger_id=uuid.uuid4(), + enrollment_id=job.enrollment_id, + action=job.action, + applied_at=timestamp, + affected_events=job.affected_events, + evidence_digest=digest, + ) + + +def _failure_execution( + job: RetentionJob, error: Exception, *, now: datetime, max_attempts: int +) -> RetentionExecution: + retryable = job.attempts < max_attempts + state = RetentionJobState.RETRYABLE if retryable else RetentionJobState.FAILED + reason = ( + RetentionReasonCode.PARTIAL_FAILURE + if retryable + else RetentionReasonCode.RETRY_EXHAUSTED + ) + evidence = _failure_evidence(job, error, now) + failed = job.model_copy( + update={ + "state": state, + "completed_at": None, + "last_error": type(error).__name__, + "evidence_digest": evidence.evidence_digest, + "reason": reason, + } + ) + return RetentionExecution(job=failed, evidence=evidence, reused=False) + + +def enqueue_for_enrollment( + session: Any, + enrollment_id: UUID, + action: RetentionAction, + *, + store_api: Optional[Any] = None, + now: Optional[datetime] = None, +) -> RetentionJob: + """Get or create the single retention job for ``(enrollment, action)``.""" + api = _resolve_store(session, store_api) + existing = api.get_job_for_enrollment_action(enrollment_id, action) + if existing is not None: + return existing + job = RetentionJob( + job_id=uuid.uuid4(), + enrollment_id=enrollment_id, + action=action, + state=RetentionJobState.PENDING, + attempts=0, + created_at=now or datetime.now(timezone.utc), + ) + return api.upsert_job(job) + + +def run_retention_job( + session: Any, + job_id: UUID, + *, + store_api: Optional[Any] = None, + now: Optional[datetime] = None, + max_attempts: int = 3, +) -> RetentionJob: + """Execute one retention job; never raises for a step failure.""" + api = _resolve_store(session, store_api) + timestamp = now or datetime.now(timezone.utc) + job = api.get_job(job_id) + if job is None: + raise ValueError(f"retention job {job_id} not found") + if job.state == RetentionJobState.COMPLETED: + return job + + running = start_retention_job(job, now=timestamp) + api.upsert_job(running) + try: + events = api.load_events(job.enrollment_id) + execution = execute_retention( + running, + events, + job.action, + now=timestamp, + applier=_StoreApplier(api), + max_attempts=max_attempts, + ) + persisted = api.upsert_job(execution.job) + if execution.evidence is not None and not execution.reused: + api.insert_evidence(execution.evidence) + return persisted + except Exception as error: # noqa: BLE001 - any failure is recorded, not raised + execution = _failure_execution( + running, error, now=timestamp, max_attempts=max_attempts + ) + try: + api.upsert_job(execution.job) + if execution.evidence is not None: + api.insert_evidence(execution.evidence) + except Exception: # noqa: BLE001 - never crash the worker process + pass + return execution.job + + +def run_retention_for_enrollment( + session: Any, + enrollment_id: UUID, + action: RetentionAction, + *, + store_api: Optional[Any] = None, + now: Optional[datetime] = None, + max_attempts: int = 3, +) -> RetentionJob: + """Idempotently enqueue and execute the job for one enrollment.""" + api = _resolve_store(session, store_api) + job = enqueue_for_enrollment( + session, enrollment_id, action, store_api=api, now=now + ) + if job.state == RetentionJobState.COMPLETED: + return job + return run_retention_job( + session, + job.job_id, + store_api=api, + now=now, + max_attempts=max_attempts, + ) diff --git a/src/research/runtime/__init__.py b/src/research/runtime/__init__.py new file mode 100644 index 0000000..991f1ab --- /dev/null +++ b/src/research/runtime/__init__.py @@ -0,0 +1,13 @@ +"""Research runtime: assignment, bootstrap and session lifecycle. + +Merged home for the enrollment-scoped assignment service, the +secret-free bootstrap manifest and capability issuance, and the research +session/agent-run state machine. The individual sub-packages keep their +original public surfaces: + +* :mod:`research.runtime.assignment` +* :mod:`research.runtime.bootstrap` +* :mod:`research.runtime.sessions` + +The core packages never import ``App``, FastAPI, or a session factory. +""" diff --git a/src/research/runtime/assignment/__init__.py b/src/research/runtime/assignment/__init__.py new file mode 100644 index 0000000..4aa50d6 --- /dev/null +++ b/src/research/runtime/assignment/__init__.py @@ -0,0 +1,37 @@ +"""Enrollment-scoped profile assignment (Issue 05). + +Public surface: + +* :mod:`research.runtime.assignment.enums` - allocation outcomes and typed + reason codes. +* :mod:`research.runtime.assignment.models` - ``AssignmentV1`` and typed + results. +* :mod:`research.runtime.assignment.service` - server-authoritative, sticky, + equal-probability allocation over ``enrollment_id``. +* :mod:`research.runtime.assignment.hashing` - the salted-hash draw and the + study's frozen assignment policy. +* :mod:`research.runtime.assignment.store` - persistence adapters taking a + caller-supplied SQLAlchemy ``Session``. + +The core package never imports ``App``, FastAPI, or a session factory. +""" + +from .enums import ( + AllocationOutcome, + AssignmentReasonCode, +) +from .models import ( + AssignmentIssue, + AssignmentResult, + AssignmentV1, +) +from .service import allocate + +__all__ = [ + "AllocationOutcome", + "AssignmentIssue", + "AssignmentReasonCode", + "AssignmentResult", + "AssignmentV1", + "allocate", +] diff --git a/src/research/runtime/assignment/enums.py b/src/research/runtime/assignment/enums.py new file mode 100644 index 0000000..0254aa5 --- /dev/null +++ b/src/research/runtime/assignment/enums.py @@ -0,0 +1,42 @@ +"""Closed vocabularies for enrollment-scoped assignment (Issue 05). + +Every member is part of a persisted contract (assignment rows, API responses), +so members are additive only. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = [ + "AllocationOutcome", + "AssignmentReasonCode", +] + + +class AllocationOutcome(str, Enum): + """Terminal result of an allocation attempt.""" + + CREATED = "CREATED" + EXISTING = "EXISTING" + INELIGIBLE = "INELIGIBLE" + INSUFFICIENT_EVIDENCE = "INSUFFICIENT_EVIDENCE" + CONFLICT = "CONFLICT" + + +class AssignmentReasonCode(str, Enum): + """Stable machine-readable reason for an allocation outcome.""" + + OK = "OK" + NO_ENROLLMENT = "NO_ENROLLMENT" + ENROLLMENT_NOT_ACTIVE = "ENROLLMENT_NOT_ACTIVE" + CONSENT_REQUIRED = "CONSENT_REQUIRED" + REVOKED = "REVOKED" + STUDY_MISMATCH = "STUDY_MISMATCH" + NO_PROFILES = "NO_PROFILES" + PROFILE_MISMATCH = "PROFILE_MISMATCH" + STICKY_EXISTING = "STICKY_EXISTING" + RANDOM_EQUAL = "RANDOM_EQUAL" + DETERMINISTIC_HASH = "DETERMINISTIC_HASH" + UNKNOWN_STRATEGY = "UNKNOWN_STRATEGY" + diff --git a/src/research/runtime/assignment/hashing.py b/src/research/runtime/assignment/hashing.py new file mode 100644 index 0000000..80575b1 --- /dev/null +++ b/src/research/runtime/assignment/hashing.py @@ -0,0 +1,92 @@ +"""Salted-hash arm assignment and the study's assignment policy. + +A study created with ``research_config_json["assignment"]["strategy"] == +"DETERMINISTIC_HASH"`` gives each enrollment the arm with index + + k = floor(K * h / 2**64) + h = first 8 bytes (big-endian) of + SHA-256("{study_id}:{randomization_epoch}:{enrollment_id}") + +over its K arms ordered by ``selection_order`` (UUIDs in lowercase hyphenated +form). The draw is reproducible from exported data, and the study id salts it, +so assignments in different studies are independent. Studies created before +this protocol decision carry no ``assignment`` block and keep the +``RANDOM_EQUAL`` draw. See ``docs/RESEARCH_ANALYTICS_SCOPE.md``. +""" + +from __future__ import annotations + +import hashlib +import uuid +from collections.abc import Mapping, Sequence +from typing import Any, TypeVar + +from research.study.protocol.enums import AssignmentStrategy + +__all__ = [ + "assignment_policy", + "assignment_strategy", + "hash_arm_index", + "hashed_profile", + "manual_override_enabled", + "new_study_assignment_policy", +] + +_Profile = TypeVar("_Profile") + + +def hash_arm_index( + study_id: uuid.UUID | str, + randomization_epoch: int, + enrollment_id: uuid.UUID | str, + arm_count: int, +) -> int: + """The arm index in ``range(arm_count)`` for one enrollment.""" + if arm_count < 1: + raise ValueError("a study needs at least one arm") + material = f"{uuid.UUID(str(study_id))}:{int(randomization_epoch)}:{uuid.UUID(str(enrollment_id))}" + digest = hashlib.sha256(material.encode("ascii")).digest() + return (arm_count * int.from_bytes(digest[:8], "big")) >> 64 + + +def _selection_key(profile: Any) -> tuple[int, str]: + profile_id = getattr(profile, "agent_profile_id", None) or getattr(profile, "profile_id", "") + return (int(getattr(profile, "selection_order", 0) or 0), str(profile_id)) + + +def hashed_profile( + profiles: Sequence[_Profile], + *, + study_id: uuid.UUID | str, + enrollment_id: uuid.UUID | str, + randomization_epoch: int = 0, +) -> _Profile: + """Pick the study arm the hash assigns (arms ordered by ``selection_order``).""" + ordered = sorted(profiles, key=_selection_key) + index = hash_arm_index(study_id, randomization_epoch, enrollment_id, len(ordered)) + return ordered[index] + + +def assignment_policy(config: Any) -> Mapping[str, Any]: + """The frozen ``assignment`` block of a study config (empty for legacy studies).""" + policy = config.get("assignment") if isinstance(config, Mapping) else None + return policy if isinstance(policy, Mapping) else {} + + +def assignment_strategy(config: Any) -> str: + """The study's allocation strategy; studies without a policy draw ``RANDOM_EQUAL``.""" + strategy = assignment_policy(config).get("strategy") + return str(strategy) if strategy else AssignmentStrategy.RANDOM_EQUAL.value + + +def manual_override_enabled(config: Any) -> bool: + """Whether the study's protocol allows the owner to change an arm by hand.""" + return assignment_policy(config).get("manual_override") is True + + +def new_study_assignment_policy(*, manual_override: bool) -> dict[str, Any]: + """The assignment block every newly created study freezes into its config.""" + return { + "strategy": AssignmentStrategy.DETERMINISTIC_HASH.value, + "manual_override": bool(manual_override), + } diff --git a/src/research/runtime/assignment/models.py b/src/research/runtime/assignment/models.py new file mode 100644 index 0000000..38fe1a8 --- /dev/null +++ b/src/research/runtime/assignment/models.py @@ -0,0 +1,74 @@ +"""Pydantic v2 contracts for enrollment-scoped profile assignment. + +:class:`AssignmentV1` is the server-authoritative, sticky agent profile +allocated for one enrollment. It is never re-randomized; launch outcomes are +recorded against ``agent_run_id`` in the canonical run/event model. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Any, Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import ( + AllocationOutcome, + AssignmentReasonCode, +) + +_FROZEN = ConfigDict(extra="forbid", frozen=True) +_BASE = ConfigDict(extra="forbid") + + +class AssignmentV1(BaseModel): + """The immutable agent profile assigned to one enrollment.""" + + model_config = _FROZEN + + assignment_id: UUID + enrollment_id: UUID + study_id: UUID + agent_profile_id: UUID + strategy: str + randomization_epoch: int = 0 + profile_digest: str + profile_snapshot_json: dict[str, Any] + status: str = "ACTIVE" + assigned_at: datetime + + +class StudyProfileSelection(BaseModel): + """A study-owned, digest-pinned profile available for random assignment.""" + + model_config = _FROZEN + + study_id: UUID + agent_profile_id: UUID + profile_digest: str + profile_snapshot_json: dict[str, Any] + selection_order: int = 0 + + +class AssignmentIssue(BaseModel): + """One typed assignment rejection reason.""" + + model_config = _BASE + + code: AssignmentReasonCode + message: str + field: str = "" + + +class AssignmentResult(BaseModel): + """Typed outcome of an allocation attempt.""" + + model_config = _BASE + + outcome: AllocationOutcome + assignment: Optional[AssignmentV1] = None + created: bool = False + reason: AssignmentReasonCode = AssignmentReasonCode.OK + issue: Optional[AssignmentIssue] = None + diff --git a/src/research/runtime/assignment/service.py b/src/research/runtime/assignment/service.py new file mode 100644 index 0000000..e451408 --- /dev/null +++ b/src/research/runtime/assignment/service.py @@ -0,0 +1,156 @@ +"""Pure server-authoritative allocation over enrollment-scoped units. + +The unit of assignment is ``enrollment_id`` (never account, device, task, or +process). Assignments are always sticky: a repeated bootstrap returns the +existing profile and never re-randomizes it. Profiles are selected by the study +at creation time and carry a digest-pinned, non-secret snapshot. The draw follows +the study's frozen assignment policy (see :mod:`.hashing`). +""" + +from __future__ import annotations + +import random +import uuid +from collections.abc import Sequence +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Optional + +from research.participants.enums import EnrollmentStatus +from research.study.protocol.enums import AssignmentStrategy + +from .enums import AllocationOutcome, AssignmentReasonCode +from .hashing import hashed_profile +from .models import AssignmentIssue, AssignmentResult, AssignmentV1, StudyProfileSelection + +if TYPE_CHECKING: + from research.participants.models import Enrollment + + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def _issue(code: AssignmentReasonCode, message: str, field: str = "") -> AssignmentIssue: + return AssignmentIssue(code=code, message=message, field=field) + + +def _blocked( + outcome: AllocationOutcome, code: AssignmentReasonCode, message: str, field: str = "" +) -> AssignmentResult: + return AssignmentResult( + outcome=outcome, reason=code, issue=_issue(code, message, field) + ) + + +def allocate( + enrollment: Enrollment, + profiles: Sequence[StudyProfileSelection], + existing: Optional[AssignmentV1] = None, + rng: Optional[random.Random] = None, + now: Optional[datetime] = None, + strategy: str = AssignmentStrategy.RANDOM_EQUAL.value, +) -> AssignmentResult: + """Allocate (or return) one sticky profile with equal probability. + + ``strategy`` is the study's frozen policy (``hashing.assignment_strategy``): + ``RANDOM_EQUAL`` draws from the system CSPRNG, ``DETERMINISTIC_HASH`` takes + the salted-hash arm; anything else is refused. + """ + timestamp = _now(now) + + if enrollment is None: + return _blocked( + AllocationOutcome.INELIGIBLE, + AssignmentReasonCode.NO_ENROLLMENT, + "an enrollment is required for allocation", + "enrollment_id", + ) + + if enrollment.status != EnrollmentStatus.ACTIVE: + return _blocked( + AllocationOutcome.INELIGIBLE, + AssignmentReasonCode.ENROLLMENT_NOT_ACTIVE, + f"enrollment is {enrollment.status.value}", + "status", + ) + + if profiles is None: + return _blocked( + AllocationOutcome.INSUFFICIENT_EVIDENCE, + AssignmentReasonCode.NO_PROFILES, + "study profiles are required for allocation", + "profiles", + ) + + if existing is not None: + if ( + existing.enrollment_id == enrollment.enrollment_id + and existing.study_id == enrollment.study_id + ): + return AssignmentResult( + outcome=AllocationOutcome.EXISTING, + assignment=existing, + created=False, + reason=AssignmentReasonCode.STICKY_EXISTING, + ) + return _blocked( + AllocationOutcome.CONFLICT, + AssignmentReasonCode.STUDY_MISMATCH, + "an assignment already exists for a different enrollment/study", + "study_id", + ) + + normalized_profiles = [StudyProfileSelection.model_validate(profile) for profile in profiles] + if not normalized_profiles: + return _blocked( + AllocationOutcome.INSUFFICIENT_EVIDENCE, + AssignmentReasonCode.NO_PROFILES, + "the study declares no agent profiles", + "profiles", + ) + if any(profile.study_id != enrollment.study_id for profile in normalized_profiles): + return _blocked( + AllocationOutcome.INSUFFICIENT_EVIDENCE, + AssignmentReasonCode.STUDY_MISMATCH, + "all profiles must belong to the enrollment study", + "study_id", + ) + + randomization_epoch = 0 + if strategy == AssignmentStrategy.DETERMINISTIC_HASH.value: + selected = hashed_profile( + normalized_profiles, + study_id=enrollment.study_id, + enrollment_id=enrollment.enrollment_id, + randomization_epoch=randomization_epoch, + ) + reason = AssignmentReasonCode.DETERMINISTIC_HASH + elif strategy == AssignmentStrategy.RANDOM_EQUAL.value: + selected = (rng or random.SystemRandom()).choice(normalized_profiles) + reason = AssignmentReasonCode.RANDOM_EQUAL + else: + return _blocked( + AllocationOutcome.INSUFFICIENT_EVIDENCE, + AssignmentReasonCode.UNKNOWN_STRATEGY, + f"unknown assignment strategy {strategy!r}", + "strategy", + ) + + assignment = AssignmentV1( + assignment_id=uuid.uuid4(), + enrollment_id=enrollment.enrollment_id, + study_id=enrollment.study_id, + agent_profile_id=selected.agent_profile_id, + strategy=strategy, + randomization_epoch=randomization_epoch, + assigned_at=timestamp, + profile_digest=selected.profile_digest, + profile_snapshot_json=selected.profile_snapshot_json, + ) + return AssignmentResult( + outcome=AllocationOutcome.CREATED, + assignment=assignment, + created=True, + reason=reason, + ) diff --git a/src/research/runtime/assignment/store.py b/src/research/runtime/assignment/store.py new file mode 100644 index 0000000..3203314 --- /dev/null +++ b/src/research/runtime/assignment/store.py @@ -0,0 +1,115 @@ +"""CRUD-style persistence helpers for assignment. + +These functions take a caller-managed SQLAlchemy ``Session`` so the core package +never imports ``App`` or touches the application singleton. Assignments are +immutable facts; the unique ``enrollment_id`` constraint is +the concurrency guard, so a concurrent first-bootstrap race yields one row and +the loser re-reads it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional, Sequence + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError + +if TYPE_CHECKING: + import uuid + + from sqlalchemy.orm import Session + + from .models import AssignmentV1 + +from database.research_schemas import StudyAssignment + + +def create_assignment( + session: Session, assignment: AssignmentV1, *, commit: bool = True +) -> StudyAssignment: + """Insert one immutable assignment row, or return the existing winner. + + The unique ``enrollment_id`` constraint is the + concurrency guard: a savepoint around the insert means a losing concurrent + create rolls back only its own insert and returns the winning row, so the + caller always uses the authoritative sticky assignment. ``commit=False`` lets + bootstrap own one unit of work. + """ + row = StudyAssignment( + assignment_id=assignment.assignment_id, + enrollment_id=assignment.enrollment_id, + study_id=assignment.study_id, + agent_profile_id=assignment.agent_profile_id, + strategy=assignment.strategy, + randomization_epoch=assignment.randomization_epoch, + profile_digest=assignment.profile_digest, + profile_snapshot_json=assignment.profile_snapshot_json, + status=assignment.status, + assigned_at=assignment.assigned_at, + ) + try: + with session.begin_nested(): + session.add(row) + except IntegrityError: + existing = get_assignment_for_enrollment(session, assignment.enrollment_id) + if existing is not None: + return existing + raise + if commit: + session.commit() + else: + session.flush() + session.refresh(row) + return row + + +def get_assignment( + session: Session, assignment_id: uuid.UUID +) -> Optional[StudyAssignment]: + """Fetch an assignment by id, or ``None``.""" + return session.get(StudyAssignment, assignment_id) + + +def get_assignment_for_enrollment( + session: Session, enrollment_id: uuid.UUID, *, lock: bool = False +) -> Optional[StudyAssignment]: + """Fetch the unique sticky assignment for one enrollment. + + ``lock`` takes a share lock until the transaction ends, so a session start + and a manual arm change (which locks the row for update) serialize. + """ + statement = select(StudyAssignment).where( + StudyAssignment.enrollment_id == enrollment_id, + ) + if lock: + statement = statement.with_for_update(read=True) + return session.execute(statement).scalars().first() + + +def list_assignments( + session: Session, enrollment_id: Optional[uuid.UUID] = None +) -> Sequence[StudyAssignment]: + """List assignments, optionally for one enrollment.""" + statement = select(StudyAssignment) + if enrollment_id is not None: + statement = statement.where(StudyAssignment.enrollment_id == enrollment_id) + statement = statement.order_by(StudyAssignment.assigned_at.asc()) + return list(session.execute(statement).scalars().all()) + + +def row_to_assignment(row: StudyAssignment) -> AssignmentV1: + """Rehydrate an assignment row into the domain model.""" + from .models import AssignmentV1 as AssignmentV1Model + + return AssignmentV1Model( + assignment_id=row.assignment_id, + enrollment_id=row.enrollment_id, + study_id=row.study_id, + agent_profile_id=row.agent_profile_id, + strategy=row.strategy, + randomization_epoch=row.randomization_epoch, + assigned_at=row.assigned_at, + profile_digest=row.profile_digest, + profile_snapshot_json=row.profile_snapshot_json, + status=row.status, + ) diff --git a/src/research/runtime/bootstrap/__init__.py b/src/research/runtime/bootstrap/__init__.py new file mode 100644 index 0000000..af070b9 --- /dev/null +++ b/src/research/runtime/bootstrap/__init__.py @@ -0,0 +1,95 @@ +"""Bootstrap manifest and session capability (Issue 05). + +Public surface: + +* :mod:`research.runtime.bootstrap.models` - ``BootstrapManifestV1`` and its sub-objects, + ``SessionCapability``, typed reason codes and results. +* :mod:`research.runtime.bootstrap.signer` - dependency-free manifest digest + HMAC + signature and verification. +* :mod:`research.runtime.bootstrap.capability` - scoped, short-lived capability issuance + and server-side verification. +* :mod:`research.runtime.bootstrap.service` - :func:`compose_bootstrap`, which validates + eligibility/compatibility before projecting and signing a secret-free manifest. +* :mod:`research.runtime.bootstrap.store` - persistence adapters taking a caller-supplied + SQLAlchemy ``Session``. + +The core package never imports ``App``, FastAPI, or a session factory. +""" + +from .capability import ( + DEFAULT_AUDIENCE, + DEFAULT_SCOPE, + compute_capability_signature, + issue_capability, + verify_capability, +) +from .models import ( + BootstrapAgentRelease, + BootstrapAssignment, + BootstrapCompatibility, + BootstrapIssue, + BootstrapManifestV1, + BootstrapOutcome, + BootstrapPolicies, + BootstrapPrivacyPolicy, + BootstrapReasonCode, + BootstrapResult, + BootstrapSessionPolicy, + BootstrapTelemetryPolicy, + CapabilityReasonCode, + CapabilityVerification, + ManifestReasonCode, + ManifestSignature, + ManifestVerification, + ResearchSessionRef, + SessionCapability, +) +from .service import ( + BootstrapSigningContext, + EphemeralSessionFactory, + SessionFactory, + compose_bootstrap, +) +from .signer import ( + canonical_manifest_bytes, + compute_manifest_digest, + manifest_digest_payload, + sign_manifest, + verify_manifest, +) + +__all__ = [ + "DEFAULT_AUDIENCE", + "DEFAULT_SCOPE", + "BootstrapAgentRelease", + "BootstrapAssignment", + "BootstrapCompatibility", + "BootstrapIssue", + "BootstrapManifestV1", + "BootstrapOutcome", + "BootstrapPolicies", + "BootstrapPrivacyPolicy", + "BootstrapReasonCode", + "BootstrapResult", + "BootstrapSessionPolicy", + "BootstrapSigningContext", + "BootstrapTelemetryPolicy", + "CapabilityReasonCode", + "CapabilityVerification", + "EphemeralSessionFactory", + "ManifestReasonCode", + "ManifestSignature", + "ManifestVerification", + "ResearchSessionRef", + "SessionCapability", + "SessionFactory", + "canonical_manifest_bytes", + "compose_bootstrap", + "compute_capability_signature", + "compute_manifest_digest", + "issue_capability", + "manifest_digest_payload", + "sign_manifest", + "verify_capability", + "verify_manifest", +] diff --git a/src/research/runtime/bootstrap/capability.py b/src/research/runtime/bootstrap/capability.py new file mode 100644 index 0000000..8fbdbc0 --- /dev/null +++ b/src/research/runtime/bootstrap/capability.py @@ -0,0 +1,235 @@ +"""Scoped, short-lived session capabilities. + +A capability is signed with HMAC-SHA256 over its canonical mapping (excluding +its own signature) and is evaluated server-side for expiry, audience, scope and +revocation epoch. It never carries a provider or account secret. +""" + +from __future__ import annotations + +import hashlib +import hmac +import uuid +from datetime import datetime, timedelta, timezone +from typing import Optional + +from research.canonical import canonical_hash + +from .models import ( + CapabilityReasonCode, + CapabilityVerification, + SessionCapability, +) + +__all__ = [ + "INFERENCE_AUDIENCE", + "INFERENCE_SCOPE", + "capability_payload", + "compute_capability_signature", + "issue_capability", + "issue_inference_capability", + "verify_capability", +] + +DEFAULT_AUDIENCE = "research-runtime" +DEFAULT_SCOPE = ("telemetry:write", "session:heartbeat", "session:close") + +#: The inference capability a Goose study arm presents to the research +#: inference gateway (``/api/research/inference/v1/chat/completions``). Same +#: signing, binding and revocation rules as the session capability; a distinct +#: audience/scope so neither capability can be replayed as the other. +INFERENCE_AUDIENCE = "inference" +INFERENCE_SCOPE = ("inference:relay",) + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def capability_payload(capability: SessionCapability) -> dict: + """Return the canonical mapping covered by the capability signature.""" + data = capability.model_dump(mode="json") + data.pop("signature", None) + return data + + +def _signature_for(capability: SessionCapability, secret: str) -> str: + digest = canonical_hash(capability_payload(capability)) + return hmac.new( + secret.encode("utf-8"), digest.encode("utf-8"), hashlib.sha256 + ).hexdigest() + + +def compute_capability_signature(capability: SessionCapability, secret: str) -> str: + """Return the expected HMAC signature for a capability.""" + return _signature_for(capability, secret) + + +def issue_capability( + audience: str, + scope: list[str], + ttl_seconds: int, + revocation_epoch: int, + secret: str, + now: Optional[datetime] = None, + *, + enrollment_id: uuid.UUID, + research_session_id: uuid.UUID, + study_id: uuid.UUID, +) -> SessionCapability: + """Issue a signed, short-lived capability bound to a subject. + + ``enrollment_id``, ``research_session_id`` and ``study_id`` are required + and covered by the HMAC, so the capability is only usable for the exact + enrollment/session/revision it was issued for. There is no default secret: + issuing without a configured signing secret is refused. + """ + if ttl_seconds <= 0: + raise ValueError("ttl_seconds must be positive") + if not secret or not secret.strip(): + raise ValueError("a non-empty signing secret is required to issue a capability") + + issued_at = _now(now) + capability = SessionCapability( + capability_id=uuid.uuid4(), + audience=audience, + scope=sorted(set(scope)), + issued_at=issued_at, + expires_at=issued_at + timedelta(seconds=ttl_seconds), + revocation_epoch=revocation_epoch, + enrollment_id=enrollment_id, + research_session_id=research_session_id, + study_id=study_id, + signature="", + ) + return capability.model_copy(update={"signature": _signature_for(capability, secret)}) + + +def issue_inference_capability( + *, + secret: str, + ttl_seconds: int, + revocation_epoch: int, + enrollment_id: uuid.UUID, + research_session_id: uuid.UUID, + study_id: uuid.UUID, + now: Optional[datetime] = None, +) -> SessionCapability: + """Issue the gateway bearer capability (audience ``inference``).""" + return issue_capability( + INFERENCE_AUDIENCE, + list(INFERENCE_SCOPE), + ttl_seconds, + revocation_epoch, + secret, + now, + enrollment_id=enrollment_id, + research_session_id=research_session_id, + study_id=study_id, + ) + + +def verify_capability( + capability: SessionCapability, + secret: str, + expected_audience: str, + expected_scope: list[str], + now: Optional[datetime] = None, + current_revocation_epoch: Optional[int] = None, + *, + expected_enrollment_id: Optional[uuid.UUID] = None, + expected_research_session_id: Optional[uuid.UUID] = None, + expected_study_id: Optional[uuid.UUID] = None, +) -> CapabilityVerification: + """Verify a capability server-side, returning a typed reason. + + The subject arguments are optional so a stateless signature/expiry check can + still be performed without a resolved resource, but every authorization use + passes them so a valid capability for another subject can never be replayed. + """ + if not secret or not secret.strip(): + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.SIGNING_SECRET_MISSING, + message="no signing secret is configured; refusing to verify capabilities", + ) + + if not capability.signature or not hmac.compare_digest( + capability.signature, _signature_for(capability, secret) + ): + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.SIGNATURE_MISMATCH, + message="capability signature does not match", + ) + + if ( + expected_enrollment_id is not None + and capability.enrollment_id != expected_enrollment_id + ): + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.SUBJECT_MISMATCH, + message="capability belongs to a different enrollment", + ) + + if ( + expected_research_session_id is not None + and capability.research_session_id != expected_research_session_id + ): + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.SESSION_MISMATCH, + message="capability belongs to a different research session", + ) + + if expected_study_id is not None and capability.study_id != expected_study_id: + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.STUDY_MISMATCH, + message="capability is bound to a different study", + ) + + if capability.audience != expected_audience: + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.WRONG_AUDIENCE, + message=( + f"capability audience {capability.audience!r} does not match " + f"{expected_audience!r}" + ), + ) + + missing = sorted(set(expected_scope) - set(capability.scope)) + if missing: + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.SCOPE_MISSING, + message=f"capability is missing required scope {missing}", + ) + + timestamp = _now(now) + if timestamp < capability.issued_at: + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.NOT_YET_VALID, + message="capability is not valid yet", + ) + if timestamp >= capability.expires_at: + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.EXPIRED, + message="capability has expired", + ) + + if ( + current_revocation_epoch is not None + and capability.revocation_epoch != current_revocation_epoch + ): + return CapabilityVerification( + ok=False, + reason=CapabilityReasonCode.REVOKED, + message="capability revocation epoch is stale", + ) + + return CapabilityVerification(ok=True, reason=CapabilityReasonCode.OK) diff --git a/src/research/runtime/bootstrap/models.py b/src/research/runtime/bootstrap/models.py new file mode 100644 index 0000000..10e829f --- /dev/null +++ b/src/research/runtime/bootstrap/models.py @@ -0,0 +1,385 @@ +"""Pydantic v2 contracts for the bootstrap manifest and session capability. + +The manifest is a signed, short-lived, secret-free projection. It contains only +the pinned revision reference, the sticky assignment, the pinned agent release, +the policy set, an optional compatibility receipt reference, and a scoped +session capability. It never contains account identity, provider credentials, +raw consent documents, or arbitrary launch commands; ``null`` optional policy +values mean inherited/disabled, and unavailable capabilities carry a typed +state and reason. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from enum import Enum +from typing import Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from research.telemetry.enums import CoverageState +from research.study.agents.models import DistributionArtifact + +_FROZEN = ConfigDict(extra="forbid", frozen=True) +_BASE = ConfigDict(extra="forbid") + + +class BootstrapOutcome(str, Enum): + """Terminal result of composing a bootstrap manifest.""" + + ISSUED = "ISSUED" + BLOCKED = "BLOCKED" + + +class BootstrapReasonCode(str, Enum): + """Stable machine-readable reason a bootstrap was blocked (or issued).""" + + OK = "OK" + ENROLLMENT_NOT_ACTIVE = "ENROLLMENT_NOT_ACTIVE" + CONSENT_REQUIRED = "CONSENT_REQUIRED" + REVOKED = "REVOKED" + STUDY_NOT_OPEN = "STUDY_NOT_OPEN" + STUDY_CLOSED = "STUDY_CLOSED" + STUDY_MISMATCH = "STUDY_MISMATCH" + ASSIGNMENT_MISMATCH = "ASSIGNMENT_MISMATCH" + RELEASE_NOT_QUALIFIED = "RELEASE_NOT_QUALIFIED" + RELEASE_NOT_FOUND = "RELEASE_NOT_FOUND" + ARTIFACT_UNAVAILABLE = "ARTIFACT_UNAVAILABLE" + ARTIFACT_MISMATCH = "ARTIFACT_MISMATCH" + # The release is qualified for *some* artifact, but not for the exact + # artifact/platform selected for this host (ISSUE-10). + ARTIFACT_NOT_QUALIFIED = "ARTIFACT_NOT_QUALIFIED" + COMPATIBILITY_MISSING = "COMPATIBILITY_MISSING" + INCOMPATIBLE_ENVIRONMENT = "INCOMPATIBLE_ENVIRONMENT" + CAPABILITY_INVALID = "CAPABILITY_INVALID" + # No signing secret is configured, so no capability/manifest may be issued. + SIGNING_SECRET_MISSING = "SIGNING_SECRET_MISSING" + # Operator kill switch: no new session may be bootstrapped while engaged. + KILL_SWITCH_ENGAGED = "KILL_SWITCH_ENGAGED" + # A gateway-bound release (Goose) does not bind the research inference + # gateway runtime fields, so the agent could not be pointed at it. + INFERENCE_GATEWAY_UNBOUND = "INFERENCE_GATEWAY_UNBOUND" + + +class ManifestReasonCode(str, Enum): + """Stable machine-readable manifest verification reason.""" + + OK = "OK" + MISSING_SIGNATURE = "MISSING_SIGNATURE" + MANIFEST_DIGEST_MISMATCH = "MANIFEST_DIGEST_MISMATCH" + MANIFEST_SIGNATURE_MISMATCH = "MANIFEST_SIGNATURE_MISMATCH" + + +class CapabilityReasonCode(str, Enum): + """Stable machine-readable capability verification reason.""" + + OK = "OK" + SIGNATURE_MISMATCH = "SIGNATURE_MISMATCH" + SIGNING_SECRET_MISSING = "SIGNING_SECRET_MISSING" + WRONG_AUDIENCE = "WRONG_AUDIENCE" + SCOPE_MISSING = "SCOPE_MISSING" + NOT_YET_VALID = "NOT_YET_VALID" + EXPIRED = "EXPIRED" + REVOKED = "REVOKED" + # The capability is cryptographically valid but belongs to a different + # subject than the resource it is being presented for. + SUBJECT_MISMATCH = "SUBJECT_MISMATCH" + SESSION_MISMATCH = "SESSION_MISMATCH" + STUDY_MISMATCH = "STUDY_MISMATCH" + + +class ResearchSessionRef(BaseModel): + """Opaque research session descriptor (Issue 07 owns its lifecycle).""" + + model_config = _BASE + + research_session_id: UUID + opened_at: datetime + + +class SessionCapability(BaseModel): + """Short-lived, scoped, signed session capability bound to its subject. + + The capability is only meaningful for one ``(study, enrollment, research + session)`` tuple. It carries that subject explicitly and the signed + payload covers it, so a capability issued for one participant cannot be + replayed against another participant's enrollment, session, or revision. + """ + + model_config = _FROZEN + + capability_id: UUID + audience: str + scope: list[str] = Field(default_factory=list) + issued_at: datetime + expires_at: datetime + revocation_epoch: int = 0 + # The subject the capability is bound to. All three are required so a + # capability can never be issued without an explicit binding. + enrollment_id: UUID + research_session_id: UUID + study_id: UUID + signature: str = "" + + +class BootstrapAssignment(BaseModel): + """The assignment projection embedded in a manifest.""" + + model_config = _FROZEN + + assignment_id: UUID + agent_profile_id: UUID + strategy: str + randomization_epoch: int = 0 + profile_digest: str + + +class BootstrapAgentConfigBinding(BaseModel): + """One declared BYOA profile→agent configuration translation (ISSUE-03). + + The release owns how a frozen profile field reaches a participant-installed + agent: an environment variable (``env``), an argv pair (``arg``), or the + research inference gateway's tool allowlist (``gateway``, which the plugin + applies nothing for), with an optional server-vocabulary ``value_map`` and + list ``format``. Projected so + the plugin can apply the exact contract that profile/study validation + enforced. + """ + + model_config = _FROZEN + + field: str + transport: str + key: str + format: str = "string" + value_map: dict[str, str] = Field(default_factory=dict) + + +class BootstrapAgentRelease(BaseModel): + """The pinned agent release projection embedded in a manifest. + + ``distribution_mode`` selects how the agent reaches the participant: + ``PACKAGED`` (default) pins ``artifact_digest``; ``BYOA_EXTERNAL`` pins a + participant-installed command/package identity and leaves + ``artifact_digest`` empty. + + ``adapter_id``/``adapter_version`` are the non-secret identity of the + adapter the release was qualified with. They are an opaque reference, not a + launch path: the server-side adapter allowlist resolves them, and no secret, + revision or condition field is projected. + """ + + model_config = _FROZEN + + agent_id: str + release_id: str + version: str = "" + artifact: Optional[DistributionArtifact] = None + min_plugin_version: Optional[str] = None + min_protocol_version: Optional[str] = None + max_protocol_version: Optional[str] = None + # The bootstrap pin is the ZIP fingerprint: ``artifact_digest``/``archive_sha256`` + # are the same value for a PACKAGED release and empty for BYOA. + artifact_digest: str = "" + archive_sha256: Optional[str] = None + # The executable name inside the pinned archive (not its bytes). + executable: Optional[str] = None + adapter_digest: Optional[str] = None + adapter_id: Optional[str] = None + adapter_version: Optional[str] = None + distribution_mode: str = "PACKAGED" + agent_command: Optional[str] = None + agent_command_args: list[str] = Field(default_factory=list) + agent_package: Optional[str] = None + config_bindings: list[BootstrapAgentConfigBinding] = Field(default_factory=list) + + +class BootstrapAgentProfile(BaseModel): + """The non-secret provider/model identity of a condition's agent profile. + + This is a projection of the server-side ``AgentProfile`` the condition + references. It deliberately carries no credential and not even the + ``api_key_ref`` env-var *name*: the runtime relays inference through the + backend, which resolves and injects the key server-side, so the secret and + its reference never leave the server. + """ + + model_config = _FROZEN + + profile_id: UUID + name: str + framework_version: str + model: str + base_url: Optional[str] = None + temperature: Optional[float] = None + # Frozen executable fields a BYOA release may translate (ISSUE-03 Path A). + tools_json: str = "[]" + approval_policy: str = "auto" + max_steps: int = 1 + + +class BootstrapTelemetryPolicy(BaseModel): + """Telemetry field-class allowance projected into a manifest.""" + + model_config = _FROZEN + + allowed_field_classes: list[str] = Field(default_factory=list) + content_capture: bool = False + # Whether the participant's consent is currently active, derived server-side + # from the enrollment (D-1). The client mirrors it and never invents consent. + consent_active: bool = False + + +class BootstrapPrivacyPolicy(BaseModel): + """Retention policy projected into a manifest.""" + + model_config = _FROZEN + + retention_action: str + retention_days: Optional[int] = None + + +class BootstrapSessionPolicy(BaseModel): + """Session timing policy projected into a manifest.""" + + model_config = _FROZEN + + idle_timeout_seconds: Optional[int] = None + resume_grace_seconds: Optional[int] = None + heartbeat_seconds: Optional[int] = None + + +class BootstrapPolicies(BaseModel): + """The policy set; ``null`` members mean inherited/disabled.""" + + model_config = _FROZEN + + telemetry_policy: Optional[BootstrapTelemetryPolicy] = None + privacy_policy: Optional[BootstrapPrivacyPolicy] = None + session_policy: Optional[BootstrapSessionPolicy] = None + consent_policy_digest: Optional[str] = None + + +class BootstrapCompatibility(BaseModel): + """Compatibility projection with an explicit coverage state and reason.""" + + model_config = _FROZEN + + receipt_ref: Optional[str] = None + state: CoverageState = CoverageState.UNKNOWN + reason: Optional[str] = None + + +#: Relative path (no leading slash: the plugin's manifest scanner rejects +#: absolute-path strings) Goose joins to the research server origin. +INFERENCE_GATEWAY_BASE_PATH = "api/research/inference/v1/chat/completions" + + +class BootstrapInferenceGateway(BaseModel): + """How a gateway-bound agent (Goose) reaches the study's provider key. + + The manifest carries no gateway origin on purpose: the plugin points the + agent at the origin it bootstrapped from, so a manifest can never redirect + prompts elsewhere. ``capability`` is the inference capability (audience + ``inference``); the plugin encodes it as the agent's bearer token and hands + it over through an owner-only file, never argv. + """ + + model_config = _FROZEN + + provider_kind: str = "openai_compatible" + base_path: str = INFERENCE_GATEWAY_BASE_PATH + capability: SessionCapability + + +class BootstrapManifestV1(BaseModel): + """Signed, short-lived, secret-free launch contract (schema version 1).""" + + model_config = _FROZEN + + manifest_version: str = "1" + manifest_digest: str = "" + signature: str = "" + generated_at: datetime + study_id: UUID + enrollment_id: UUID + research_config_digest: Optional[str] = None + research_session: ResearchSessionRef + assignment: BootstrapAssignment + agent_release: BootstrapAgentRelease + # Optional non-secret provider/model identity of the condition's linked + # agent profile. Absent (null) for protocols whose conditions predate the + # profile link or that pin a release without a profile. + agent_profile: Optional[BootstrapAgentProfile] = None + policies: BootstrapPolicies + compatibility_receipt_ref: Optional[str] = None + compatibility: BootstrapCompatibility = Field(default_factory=BootstrapCompatibility) + session_capability: SessionCapability + # Present only for arms whose agent calls the research inference gateway + # (Goose); ``null`` for the managed runtime (ACP relay) and for Codex. + inference_gateway: Optional[BootstrapInferenceGateway] = None + + +class ManifestSignature(BaseModel): + """Content digest and HMAC signature for a manifest.""" + + model_config = _BASE + + digest: str + signature: str + + +class ManifestVerification(BaseModel): + """Typed result of verifying a manifest's digest and signature.""" + + model_config = _BASE + + ok: bool + reason: ManifestReasonCode = ManifestReasonCode.OK + expected_digest: str = "" + actual_digest: str = "" + message: Optional[str] = None + + +class CapabilityVerification(BaseModel): + """Typed result of server-side capability verification.""" + + model_config = _BASE + + ok: bool + reason: CapabilityReasonCode = CapabilityReasonCode.OK + message: Optional[str] = None + + +class BootstrapIssue(BaseModel): + """One typed bootstrap block reason.""" + + model_config = _BASE + + code: BootstrapReasonCode + message: str + field: str = "" + + +class BootstrapPreparationV1(BaseModel): + """Authorized assignment metadata; creates no session or capability.""" + + model_config = _FROZEN + + enrollment_id: UUID + study_id: UUID + assignment: BootstrapAssignment + agent_release: BootstrapAgentRelease + + +class BootstrapResult(BaseModel): + """Typed outcome of composing a bootstrap manifest.""" + + model_config = _BASE + + outcome: BootstrapOutcome + reason: BootstrapReasonCode = BootstrapReasonCode.OK + manifest: Optional[BootstrapManifestV1] = None + preparation: Optional[BootstrapPreparationV1] = None + issue: Optional[BootstrapIssue] = None diff --git a/src/research/runtime/bootstrap/service.py b/src/research/runtime/bootstrap/service.py new file mode 100644 index 0000000..6b9541c --- /dev/null +++ b/src/research/runtime/bootstrap/service.py @@ -0,0 +1,489 @@ +"""Bootstrap composition for an active, web-enrolled study participant.""" + +from __future__ import annotations + +import json +import uuid +from datetime import datetime, timezone +from typing import Any, Callable, Mapping, Optional, Protocol + +from packaging.version import InvalidVersion, Version +from pydantic import BaseModel, ConfigDict, Field + +from research.canonical import canonical_hash +from research.compatibility.enums import CompatibilityDecision +from research.participants.enums import EnrollmentStatus +from research.study.agents.distributions import ( + missing_gateway_bindings, + release_bindings, + requires_inference_gateway, +) +from research.study.agents.models import normalize_platform +from research.telemetry.enums import CoverageState +from research.telemetry.privacy.engine import runtime_field_classes + +from .capability import issue_capability, issue_inference_capability +from .models import ( + BootstrapAgentConfigBinding, + BootstrapAgentProfile, + BootstrapAgentRelease, + BootstrapAssignment, + BootstrapCompatibility, + BootstrapInferenceGateway, + BootstrapIssue, + BootstrapManifestV1, + BootstrapOutcome, + BootstrapPolicies, + BootstrapPreparationV1, + BootstrapPrivacyPolicy, + BootstrapReasonCode, + BootstrapResult, + BootstrapSessionPolicy, + BootstrapTelemetryPolicy, + ResearchSessionRef, +) +from .signer import sign_manifest + +__all__ = [ + "BootstrapSigningContext", + "EphemeralSessionFactory", + "SessionFactory", + "compose_bootstrap", +] + +_BASE = ConfigDict(extra="forbid") + + +class SessionFactory(Protocol): + """Creates (or reuses) a research session for an enrollment and study.""" + + def create_for_enrollment( + self, + enrollment: Any, + study: Any, + now: datetime, + context_id: str = "", + ) -> ResearchSessionRef: # pragma: no cover - structural protocol + ... + + +class EphemeralSessionFactory: + """Test-only factory that returns a fresh session reference.""" + + def create_for_enrollment( + self, + enrollment: Any, + study: Any, + now: datetime, + context_id: str = "", + ) -> ResearchSessionRef: + return ResearchSessionRef(research_session_id=uuid.uuid4(), opened_at=now) + + +class BootstrapSigningContext(BaseModel): + """Injected signing parameters for manifest and capability issuance.""" + + model_config = _BASE + + secret: str + capability_ttl_seconds: int = 900 + # Goose reads its bearer once per process; liveness, budget and revocation + # are re-checked on every gateway call, so the TTL is defence in depth. + inference_capability_ttl_seconds: int = 7 * 24 * 3600 + audience: str = "research-runtime" + scope: list[str] = Field( + default_factory=lambda: [ + "telemetry:write", + "session:heartbeat", + "session:close", + ] + ) + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def _blocked( + code: BootstrapReasonCode, message: str, field: str = "" +) -> BootstrapResult: + return BootstrapResult( + outcome=BootstrapOutcome.BLOCKED, + reason=code, + issue=BootstrapIssue(code=code, message=message, field=field), + ) + + +def _study_is_open(study: Any, now: datetime) -> Optional[BootstrapIssue]: + if study is None or not getattr(study, "is_research", False): + return BootstrapIssue( + code=BootstrapReasonCode.STUDY_NOT_OPEN, + message="the study is not a research study", + field="study", + ) + if getattr(study, "research_status", None) == "STUDY_STOPPED": + return BootstrapIssue( + code=BootstrapReasonCode.STUDY_CLOSED, + message="the study has been stopped", + field="research_status", + ) + if not getattr(study, "is_active", False): + return BootstrapIssue( + code=BootstrapReasonCode.STUDY_NOT_OPEN, + message="the study is not active", + field="is_active", + ) + starts_at = getattr(study, "starts_at", None) + ends_at = getattr(study, "ends_at", None) + if starts_at is not None and now < starts_at: + return BootstrapIssue( + code=BootstrapReasonCode.STUDY_NOT_OPEN, + message="the study has not opened yet", + field="starts_at", + ) + if ends_at is not None and now > ends_at: + return BootstrapIssue( + code=BootstrapReasonCode.STUDY_CLOSED, + message="the study window has closed", + field="ends_at", + ) + return None + + +def _policies(study: Any, enrollment: Any) -> BootstrapPolicies: + config = getattr(study, "research_config_json", None) or {} + telemetry = config.get("telemetry_policy", {}) or {} + privacy = config.get("privacy_policy", {}) or {} + session = config.get("session_policy", {}) or {} + allowed = telemetry.get("allowed_field_classes", []) + # D-1: the server is the consent authority. The manifest mirrors the live + # enrollment state so the client never invents consent. + consent_active = ( + enrollment is not None + and getattr(enrollment, "status", None) == EnrollmentStatus.ACTIVE + and getattr(enrollment, "consent_accepted_at", None) is not None + ) + return BootstrapPolicies( + telemetry_policy=BootstrapTelemetryPolicy( + # Resolved to the runtime vocabulary (SYSTEM/BEHAVIORAL/...): the + # plugin and its ACP proxy do not know the study names and would + # otherwise drop the metadata those names stand for. + allowed_field_classes=runtime_field_classes(allowed), + content_capture=bool(telemetry.get("content_capture", False)), + consent_active=consent_active, + ), + privacy_policy=BootstrapPrivacyPolicy( + retention_action=privacy.get("retention_action", "RETAIN_ANONYMIZED"), + retention_days=privacy.get("retention_days"), + ), + session_policy=BootstrapSessionPolicy( + idle_timeout_seconds=session.get("idle_timeout_seconds"), + resume_grace_seconds=session.get("resume_grace_seconds"), + heartbeat_seconds=session.get("heartbeat_seconds"), + ), + ) + + +def _profile_projection(snapshot: dict[str, Any]) -> Optional[BootstrapAgentProfile]: + profile_id = snapshot.get("profile_id") + if not profile_id: + return None + return BootstrapAgentProfile( + profile_id=uuid.UUID(str(profile_id)), + name=str(snapshot.get("name", "assigned-profile")), + framework_version=str(snapshot.get("framework_version", "code4me2-agent")), + model=str(snapshot.get("model", "")), + temperature=snapshot.get("temperature"), + # ISSUE-03 Path A: the frozen executable fields a BYOA release may + # translate into the external agent's configuration. Non-secret. + tools_json=( + snapshot.get("tools_json") + if isinstance(snapshot.get("tools_json"), str) + else json.dumps(snapshot.get("tools_json") or [], separators=(",", ":")) + ), + approval_policy=str(snapshot.get("approval_policy", "auto")), + max_steps=int(snapshot.get("max_steps", 1) or 1), + ) + + +def _host_tested(release: Any, platform: Optional[tuple[str, str]]) -> bool: + if platform is None: + return True + requested = normalize_platform(*platform) + return any((result.os, result.arch) == requested for result in release.tests) + + +def compose_bootstrap( + enrollment: Any, + study: Any, + assignment: Any, + release: Any, + compatibility_ref: Optional[str], + session_factory: SessionFactory, + signer: BootstrapSigningContext, + now: Optional[datetime] = None, + *, + compatibility_result: Optional[Any] = None, + platform: Optional[tuple[str, str]] = None, + agent_profile: Optional[BootstrapAgentProfile] = None, + kill_switch_check: Optional[Callable[[], bool]] = None, + context_id: str = "", + prepare_only: bool = False, + plugin_version: Optional[str] = None, +) -> BootstrapResult: + """Compose a signed, short-lived, secret-free bootstrap manifest. + + The assigned release must be tested for the participant's ``(os, arch)``: + a release-level ``QUALIFIED`` that only covers a different platform never + issues a manifest. + """ + timestamp = _now(now) + if kill_switch_check is not None and kill_switch_check(): + return _blocked( + BootstrapReasonCode.KILL_SWITCH_ENGAGED, + "an operator kill switch is engaged for this scope", + "kill_switch", + ) + if signer is None or not signer.secret or not signer.secret.strip(): + return _blocked( + BootstrapReasonCode.SIGNING_SECRET_MISSING, + "no bootstrap signing secret is configured", + "signer.secret", + ) + if enrollment is None or enrollment.status != EnrollmentStatus.ACTIVE: + return _blocked( + BootstrapReasonCode.ENROLLMENT_NOT_ACTIVE, + "an active enrollment is required", + "enrollment_id", + ) + study_issue = _study_is_open(study, timestamp) + if study_issue is not None: + return BootstrapResult( + outcome=BootstrapOutcome.BLOCKED, + reason=study_issue.code, + issue=study_issue, + ) + if assignment is None or ( + assignment.enrollment_id != enrollment.enrollment_id + or assignment.study_id != enrollment.study_id + ): + return _blocked( + BootstrapReasonCode.ASSIGNMENT_MISMATCH, + "assignment does not belong to this enrollment/study", + "assignment_id", + ) + + snapshot = dict(assignment.profile_snapshot_json or {}) + if not assignment.profile_digest or canonical_hash(snapshot) != assignment.profile_digest: + return _blocked( + BootstrapReasonCode.ASSIGNMENT_MISMATCH, + "assignment profile digest is missing", + "profile_digest", + ) + profile = agent_profile or _profile_projection(snapshot) + if profile is None: + return _blocked( + BootstrapReasonCode.ASSIGNMENT_MISMATCH, + "assignment has no profile snapshot", + "profile_snapshot_json", + ) + if str(snapshot.get("profile_id")) != str(assignment.agent_profile_id): + return _blocked( + BootstrapReasonCode.ASSIGNMENT_MISMATCH, + "assignment profile id does not match its profile snapshot", + "agent_profile_id", + ) + if profile.profile_id != assignment.agent_profile_id: + return _blocked( + BootstrapReasonCode.ASSIGNMENT_MISMATCH, + "bootstrap profile does not match the assigned profile", + "agent_profile_id", + ) + + release_id = snapshot.get("release_id") + if release is None: + return _blocked( + BootstrapReasonCode.RELEASE_NOT_FOUND, + "the assigned profile release is not registered", + "release_id", + ) + if not release_id or str(release.release_id) != str(release_id): + return _blocked( + BootstrapReasonCode.ARTIFACT_MISMATCH, + "release does not match the assigned profile snapshot", + "release_id", + ) + if compatibility_result is not None and compatibility_result.decision != CompatibilityDecision.COMPATIBLE: + return _blocked( + BootstrapReasonCode.INCOMPATIBLE_ENVIRONMENT, + f"compatibility decision is {compatibility_result.decision.value}", + "compatibility", + ) + qualification = getattr(release, "qualification_status", None) + if getattr(qualification, "value", qualification) != "QUALIFIED": + return _blocked( + BootstrapReasonCode.RELEASE_NOT_QUALIFIED, + "the assigned release is not qualified", + "release_id", + ) + # Passing tests must cover the participant's exact platform. + if not _host_tested(release, platform): + return _blocked( + BootstrapReasonCode.RELEASE_NOT_QUALIFIED, + "the release has no passing producer tests for this platform", + "release_id", + ) + if not getattr(release, "is_byoa", False): + if platform is not None: + artifact = release.artifact_for(platform[0], platform[1]) + else: + artifacts = sorted(release.artifacts, key=lambda item: (item.os, item.arch)) + artifact = artifacts[0] if len(artifacts) == 1 else None + if artifact is None or not artifact.sha256: + return _blocked( + BootstrapReasonCode.ARTIFACT_UNAVAILABLE, + "no artifact is available for this platform", + "release.artifacts", + ) + artifact_digest = artifact.sha256 + else: + artifact_digest = "" + + if requires_inference_gateway(profile.framework_version): + gateway_missing = missing_gateway_bindings(release_bindings(release)) + if gateway_missing: + return _blocked( + BootstrapReasonCode.INFERENCE_GATEWAY_UNBOUND, + "the release does not bind the research inference gateway: " + + ", ".join(gateway_missing), + "release_id", + ) + + adapter = getattr(release, "adapter", None) + adapter_id = getattr(adapter, "adapter_id", None) + adapter_version = getattr(adapter, "version", None) + agent_release = BootstrapAgentRelease( + agent_id=release.agent_id, + release_id=release.release_id, + version=release.version, + artifact=artifact if not release.is_byoa else None, + min_plugin_version=release.min_plugin_version, + min_protocol_version=release.min_protocol_version, + max_protocol_version=release.max_protocol_version, + artifact_digest=artifact_digest, + archive_sha256=(artifact.sha256 if not release.is_byoa else None), + executable=(artifact.executable if not release.is_byoa else None), + adapter_digest=getattr(adapter, "digest", None), + adapter_id=adapter_id, + adapter_version=adapter_version, + distribution_mode=getattr(getattr(release, "distribution_mode", None), "value", "PACKAGED"), + agent_command=release.agent_command, + agent_command_args=list(release.agent_command_args), + agent_package=release.agent_package, + config_bindings=[ + BootstrapAgentConfigBinding( + field=binding.field, + transport=binding.transport, + key=binding.key, + format=binding.format, + value_map=dict(binding.value_map), + ) + for binding in (getattr(release, "byoa_config", None) or []) + ], + ) + if release.min_plugin_version: + try: + reported = plugin_version or "" + current = Version(reported.split("-", 1)[0].split("+", 1)[0]) + minimum = Version(release.min_plugin_version) + compatible_plugin = current > minimum or (current == minimum and "-" not in reported) + except InvalidVersion: + compatible_plugin = False + if not compatible_plugin: + return _blocked( + BootstrapReasonCode.INCOMPATIBLE_ENVIRONMENT, + f"Update Code4Me to {release.min_plugin_version} or later to use this study's agent.", + "plugin_version", + ) + if prepare_only: + if not _policies(study, enrollment).telemetry_policy.consent_active: + return _blocked(BootstrapReasonCode.CONSENT_REQUIRED, "study consent is required before preparing the agent", "consent") + return BootstrapResult( + outcome=BootstrapOutcome.ISSUED, + preparation=BootstrapPreparationV1( + enrollment_id=enrollment.enrollment_id, + study_id=study.study_id, + assignment=BootstrapAssignment( + assignment_id=assignment.assignment_id, + agent_profile_id=assignment.agent_profile_id, + strategy=assignment.strategy, + randomization_epoch=assignment.randomization_epoch, + profile_digest=assignment.profile_digest, + ), + agent_release=agent_release, + ), + ) + + research_session = session_factory.create_for_enrollment( + enrollment, study, timestamp, context_id + ) + capability = issue_capability( + audience=signer.audience, + scope=signer.scope, + ttl_seconds=signer.capability_ttl_seconds, + revocation_epoch=enrollment.revocation_epoch, + secret=signer.secret, + now=timestamp, + enrollment_id=enrollment.enrollment_id, + research_session_id=research_session.research_session_id, + study_id=enrollment.study_id, + ) + # Gateway-bound arms (Goose) get the research inference gateway block with + # their own inference capability; a release that cannot be pointed at the + # gateway blocks here rather than running on the participant's own key. + inference_gateway = None + if profile is not None and requires_inference_gateway(profile.framework_version): + inference_gateway = BootstrapInferenceGateway( + capability=issue_inference_capability( + secret=signer.secret, + ttl_seconds=signer.inference_capability_ttl_seconds, + revocation_epoch=enrollment.revocation_epoch, + enrollment_id=enrollment.enrollment_id, + research_session_id=research_session.research_session_id, + study_id=enrollment.study_id, + now=timestamp, + ) + ) + draft = BootstrapManifestV1( + generated_at=timestamp, + study_id=enrollment.study_id, + enrollment_id=enrollment.enrollment_id, + research_config_digest=getattr(study, "research_config_digest", None), + research_session=research_session, + assignment=BootstrapAssignment( + assignment_id=assignment.assignment_id, + agent_profile_id=assignment.agent_profile_id, + strategy=assignment.strategy, + randomization_epoch=assignment.randomization_epoch, + profile_digest=assignment.profile_digest, + ), + agent_release=agent_release, + agent_profile=profile, + policies=_policies(study, enrollment), + compatibility_receipt_ref=compatibility_ref, + compatibility=BootstrapCompatibility( + receipt_ref=compatibility_ref, + state=(CoverageState.AVAILABLE if compatibility_result is not None else CoverageState.UNAVAILABLE), + reason=None if compatibility_result is not None else "NOT_REQUIRED", + ), + session_capability=capability, + inference_gateway=inference_gateway, + ) + signature = sign_manifest(draft, signer.secret) + return BootstrapResult( + outcome=BootstrapOutcome.ISSUED, + reason=BootstrapReasonCode.OK, + manifest=draft.model_copy( + update={"manifest_digest": signature.digest, "signature": signature.signature} + ), + ) diff --git a/src/research/runtime/bootstrap/signer.py b/src/research/runtime/bootstrap/signer.py new file mode 100644 index 0000000..419c01e --- /dev/null +++ b/src/research/runtime/bootstrap/signer.py @@ -0,0 +1,113 @@ +"""Dependency-free manifest signing and verification. + +The manifest digest is content-addressed over the canonical JSON of the manifest +with ``manifest_digest`` and the capability signature excluded. The signature is +an HMAC-SHA256 over that digest using an injected secret, so signing needs no +new dependency and no PKI. Server-side verification is authoritative. +""" + +from __future__ import annotations + +import hashlib +import hmac + +from research.canonical import canonical_bytes, canonical_hash + +from .models import ( + BootstrapManifestV1, + ManifestReasonCode, + ManifestSignature, + ManifestVerification, +) + +__all__ = [ + "compute_manifest_digest", + "manifest_digest_payload", + "sign_manifest", + "verify_manifest", +] + + +def manifest_digest_payload(manifest: BootstrapManifestV1) -> dict: + """Return the canonical mapping covered by the manifest digest. + + Excludes the ``manifest_digest`` and ``signature`` fields, so the digest and + its signature are stable and self-referential content is never hashed. + """ + data = manifest.model_dump(mode="json") + data.pop("manifest_digest", None) + data.pop("signature", None) + return data + + +def compute_manifest_digest(manifest: BootstrapManifestV1) -> str: + """Return the SHA-256 digest of the digest-covered canonical mapping.""" + return canonical_hash(manifest_digest_payload(manifest)) + + +def _hmac_hex(secret: str, message: str) -> str: + return hmac.new( + secret.encode("utf-8"), message.encode("utf-8"), hashlib.sha256 + ).hexdigest() + + +def sign_manifest( + manifest: BootstrapManifestV1, secret: str +) -> ManifestSignature: + """Return ``(digest, signature)`` for a manifest.""" + digest = compute_manifest_digest(manifest) + return ManifestSignature(digest=digest, signature=_hmac_hex(secret, digest)) + + +def verify_manifest( + manifest: BootstrapManifestV1, secret: str +) -> ManifestVerification: + """Verify the manifest digest and signature, returning a typed reason.""" + expected = compute_manifest_digest(manifest) + if not manifest.manifest_digest: + return ManifestVerification( + ok=False, + reason=ManifestReasonCode.MANIFEST_DIGEST_MISMATCH, + expected_digest=expected, + actual_digest="", + message="manifest_digest is missing", + ) + if manifest.manifest_digest != expected: + return ManifestVerification( + ok=False, + reason=ManifestReasonCode.MANIFEST_DIGEST_MISMATCH, + expected_digest=expected, + actual_digest=manifest.manifest_digest, + message="manifest content does not match its digest", + ) + + signature = manifest.signature + if not signature: + return ManifestVerification( + ok=False, + reason=ManifestReasonCode.MISSING_SIGNATURE, + expected_digest=expected, + actual_digest=manifest.manifest_digest, + message="manifest is not signed", + ) + + if not hmac.compare_digest(signature, _hmac_hex(secret, expected)): + return ManifestVerification( + ok=False, + reason=ManifestReasonCode.MANIFEST_SIGNATURE_MISMATCH, + expected_digest=expected, + actual_digest=manifest.manifest_digest, + message="manifest signature does not match", + ) + + return ManifestVerification( + ok=True, + reason=ManifestReasonCode.OK, + expected_digest=expected, + actual_digest=manifest.manifest_digest, + ) + + +def canonical_manifest_bytes(manifest: BootstrapManifestV1) -> bytes: + """Convenience: canonical bytes of the manifest (signature excluded).""" + return canonical_bytes(manifest_digest_payload(manifest)) diff --git a/src/research/runtime/bootstrap/store.py b/src/research/runtime/bootstrap/store.py new file mode 100644 index 0000000..dd4da0e --- /dev/null +++ b/src/research/runtime/bootstrap/store.py @@ -0,0 +1,28 @@ +"""Persistence helpers for bootstrap session authorization. + +These functions take a caller-managed SQLAlchemy ``Session`` so the core package +never imports ``App``. Session capabilities are signed and verified statelessly +(HMAC), so no capability row is written; the authoritative revocation epoch +lives on the enrollment row and is the only server-side state consulted here. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + import uuid + + from sqlalchemy.orm import Session + +from database.research_schemas import ResearchEnrollment + + +def revocation_epoch_for( + session: Session, enrollment_id: uuid.UUID +) -> Optional[int]: + """Return the authoritative revocation epoch for an enrollment, or ``None``.""" + row = session.get(ResearchEnrollment, enrollment_id) + if row is None: + return None + return row.revocation_epoch diff --git a/src/research/runtime/sessions/__init__.py b/src/research/runtime/sessions/__init__.py new file mode 100644 index 0000000..c147a54 --- /dev/null +++ b/src/research/runtime/sessions/__init__.py @@ -0,0 +1,80 @@ +"""Research session lifecycle and agent runs (Issue 07). + +Public surface: + +* :mod:`research.runtime.sessions.enums` - session states, close reasons, agent-run + outcomes and typed reason codes. +* :mod:`research.runtime.sessions.models` - ``ResearchSessionV1``, ``AgentRunV1``, + ``SessionPolicyV1``, transitions and typed results. +* :mod:`research.runtime.sessions.service` - the pure state machine (idle/resume values + are injected study policy, never compiled constants). +* :mod:`research.runtime.sessions.store` - persistence adapters taking a caller-supplied + SQLAlchemy ``Session``. + +The core package never imports ``App``, FastAPI, or a session factory. +""" + +from .enums import AgentRunOutcome, CloseReason, SessionReasonCode, SessionState +from .models import ( + AgentRunResult, + AgentRunV1, + ResearchSessionV1, + ResumeResult, + SessionIssue, + SessionPolicyV1, + SessionResult, + SessionTransition, +) +from .service import ( + can_transition, + close, + end, + end_agent_run, + expire_if_idle, + go_offline, + on_agent_run_crashed, + on_qualifying_activity, + open_session, + policy_ref, + record_activity, + record_liveness, + recover, + resume, + revoke, + session_policy_from_study, + start_agent_run, + suspend, +) + +__all__ = [ + "AgentRunOutcome", + "AgentRunResult", + "AgentRunV1", + "CloseReason", + "ResearchSessionV1", + "ResumeResult", + "SessionIssue", + "SessionPolicyV1", + "SessionReasonCode", + "SessionResult", + "SessionState", + "SessionTransition", + "can_transition", + "close", + "end", + "end_agent_run", + "expire_if_idle", + "go_offline", + "on_agent_run_crashed", + "on_qualifying_activity", + "open_session", + "policy_ref", + "record_activity", + "record_liveness", + "recover", + "resume", + "revoke", + "session_policy_from_study", + "start_agent_run", + "suspend", +] diff --git a/src/research/runtime/sessions/enums.py b/src/research/runtime/sessions/enums.py new file mode 100644 index 0000000..3cf7c97 --- /dev/null +++ b/src/research/runtime/sessions/enums.py @@ -0,0 +1,79 @@ +"""Closed vocabularies for the research session lifecycle (Issue 07). + +Values match the Kotlin client contract in +``code4me2/src/main/kotlin/me/code4me/research/session`` so the server and the +plugin/proxy agree on the wire vocabulary. Members are additive only. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = [ + "AgentRunOutcome", + "CloseReason", + "SessionReasonCode", + "SessionState", +] + + +class SessionState(str, Enum): + """Participant activity-session lifecycle states.""" + + NOT_STARTED = "not_started" + RUNNING = "running" + OFFLINE = "offline" + SUSPENDED = "suspended" + ENDED = "ended" + REVOKED = "revoked" + + @property + def is_terminal(self) -> bool: + """Whether this state is terminal (``ENDED`` or ``REVOKED``).""" + return self in (SessionState.ENDED, SessionState.REVOKED) + + +class CloseReason(str, Enum): + """Why a research session reached a terminal state.""" + + EXPLICIT_COMPLETION = "explicit_completion" + IDLE_TIMEOUT = "idle_timeout" + RESUME_GRACE_EXPIRED = "resume_grace_expired" + IDE_CLOSED = "ide_closed" + REVOKED = "revoked" + CRASH_RECOVERED = "crash_recovered" + MIGRATED = "migrated" + # Written by the study-stop cascade (research.study.lifecycle); without + # this member every read of such a session crashed and every write + # failed as retryable instead of terminal. + STUDY_STOPPED = "STUDY_STOPPED" + + +class AgentRunOutcome(str, Enum): + """Terminal outcome of one agent run.""" + + COMPLETED = "completed" + CRASHED = "crashed" + REVOKED = "revoked" + TERMINATED = "terminated" + UNKNOWN = "unknown" + + +class SessionReasonCode(str, Enum): + """Stable machine-readable reason for a session/run operation outcome.""" + + OK = "OK" + INVALID_STATE_TRANSITION = "INVALID_STATE_TRANSITION" + SESSION_TERMINAL = "SESSION_TERMINAL" + NOT_SUSPENDED = "NOT_SUSPENDED" + RESUME_GRACE_EXPIRED = "RESUME_GRACE_EXPIRED" + IDLE_TIMEOUT = "IDLE_TIMEOUT" + POLICY_MISSING = "POLICY_MISSING" + SESSION_NOT_RUNNING = "SESSION_NOT_RUNNING" + AGENT_RUN_TERMINAL = "AGENT_RUN_TERMINAL" + ENROLLMENT_NOT_ACTIVE = "ENROLLMENT_NOT_ACTIVE" + CONSENT_REQUIRED = "CONSENT_REQUIRED" + REVOKED = "REVOKED" + CAPABILITY_INVALID = "CAPABILITY_INVALID" + KILL_SWITCH_ENGAGED = "KILL_SWITCH_ENGAGED" + STUDY_NOT_OPEN = "STUDY_NOT_OPEN" diff --git a/src/research/runtime/sessions/models.py b/src/research/runtime/sessions/models.py new file mode 100644 index 0000000..1773114 --- /dev/null +++ b/src/research/runtime/sessions/models.py @@ -0,0 +1,181 @@ +"""Pydantic v2 contracts for the research session lifecycle (Issue 07). + +A ``ResearchSessionV1`` is the authoritative participant-activity boundary. An +``AgentRunV1`` is a distinct child identity: its id is never the session id, and +an agent restart inside the resume grace creates a new run without ending the +session. Idle/resume values live in :class:`SessionPolicyV1` (a revision policy +input), never as compiled constants. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, model_validator + +from .enums import ( + AgentRunOutcome, + CloseReason, + SessionReasonCode, + SessionState, +) + +_FROZEN = ConfigDict(extra="forbid", frozen=True) +_BASE = ConfigDict(extra="forbid") + + +class SessionPolicyV1(BaseModel): + """Study-policy timing inputs for a session (never compiled constants).""" + + model_config = _BASE + + idle_timeout_seconds: int + resume_grace_seconds: int + heartbeat_seconds: int | None = None + + @model_validator(mode="after") + def _positive(self) -> SessionPolicyV1: + if self.idle_timeout_seconds <= 0: + raise ValueError("idle_timeout_seconds must be positive") + if self.resume_grace_seconds <= 0: + raise ValueError("resume_grace_seconds must be positive") + if self.heartbeat_seconds is not None and self.heartbeat_seconds <= 0: + raise ValueError("heartbeat_seconds must be positive when present") + return self + + +class ResearchSessionV1(BaseModel): + """One research session (authoritative participant-activity boundary).""" + + model_config = _FROZEN + + research_session_id: UUID + enrollment_id: UUID + study_id: UUID + # Opaque execution-context id (project/window instance); never a path. + context_id: str = "" + state: SessionState = SessionState.NOT_STARTED + opened_at: datetime | None = None + # Liveness (heartbeat) and qualifying activity are deliberately distinct: + # heartbeats prove the process is alive but must never manufacture activity + # (ISSUE-06). + last_heartbeat_at: datetime | None = None + last_activity_at: datetime | None = None + closed_at: datetime | None = None + close_reason: CloseReason | None = None + resume_generation: int = 0 + manifest_digest: str = "" + environment_ref: str | None = None + + @model_validator(mode="after") + def _consistency(self) -> ResearchSessionV1: + if self.resume_generation < 0: + raise ValueError("resume_generation must not be negative") + if self.state.is_terminal: + if self.closed_at is None or self.close_reason is None: + raise ValueError("a terminal session requires closed_at and close_reason") + elif self.closed_at is not None or self.close_reason is not None: + raise ValueError("a non-terminal session cannot have closed_at/close_reason") + return self + + +class AgentRunV1(BaseModel): + """One agent process inside a research session (distinct child identity).""" + + model_config = _FROZEN + + agent_run_id: UUID + research_session_id: UUID + agent_release_id: str | None = None + assignment_id: UUID | None = None + agent_profile_id: UUID | None = None + profile_digest: str | None = None + profile_snapshot_json: dict[str, object] | None = None + started_at: datetime + ended_at: datetime | None = None + outcome: AgentRunOutcome | None = None + + @model_validator(mode="after") + def _consistency(self) -> AgentRunV1: + if self.agent_run_id == self.research_session_id: + raise ValueError( + "AgentRun and ResearchSession identifiers must never be interchangeable" + ) + if self.ended_at is not None: + if self.ended_at < self.started_at: + raise ValueError("an agent run cannot end before it starts") + if self.outcome is None: + raise ValueError("a terminal agent run requires an outcome") + elif self.outcome is not None: + raise ValueError("a running agent run cannot have an outcome") + return self + + @property + def is_terminal(self) -> bool: + """Whether this run has ended.""" + return self.ended_at is not None + + +class SessionTransition(BaseModel): + """One recorded state transition (policy/evidence referenced).""" + + model_config = _FROZEN + + transition_id: UUID + research_session_id: UUID + from_state: SessionState + to_state: SessionState + occurred_at: datetime + reason: SessionReasonCode = SessionReasonCode.OK + close_reason: CloseReason | None = None + policy_ref: str | None = None + evidence_ref: str | None = None + + +class SessionIssue(BaseModel): + """One typed session/run rejection reason.""" + + model_config = _BASE + + code: SessionReasonCode + message: str + field: str = "" + + +class SessionResult(BaseModel): + """Typed outcome of a session operation (state + optional transition).""" + + model_config = _BASE + + accepted: bool + session: ResearchSessionV1 + transition: SessionTransition | None = None + reason: SessionReasonCode = SessionReasonCode.OK + issue: SessionIssue | None = None + + +class ResumeResult(BaseModel): + """Typed outcome of a resume attempt.""" + + model_config = _BASE + + accepted: bool + reopened: bool + requires_new_session: bool + session: ResearchSessionV1 + transition: SessionTransition | None = None + reason: SessionReasonCode = SessionReasonCode.OK + issue: SessionIssue | None = None + + +class AgentRunResult(BaseModel): + """Typed outcome of starting/ending an agent run.""" + + model_config = _BASE + + accepted: bool + run: AgentRunV1 | None = None + session: ResearchSessionV1 | None = None + reason: SessionReasonCode = SessionReasonCode.OK + issue: SessionIssue | None = None diff --git a/src/research/runtime/sessions/service.py b/src/research/runtime/sessions/service.py new file mode 100644 index 0000000..a6ef3c9 --- /dev/null +++ b/src/research/runtime/sessions/service.py @@ -0,0 +1,616 @@ +"""Pure research-session state machine (Issue 07). + +Idle and resume values are injected study-policy inputs +(:class:`~research.runtime.sessions.models.SessionPolicyV1`), never compiled constants. +Only the published lifecycle transitions are legal; every applied transition is +returned as a :class:`~research.runtime.sessions.models.SessionTransition` carrying the +reason and a policy/evidence reference. + +State diagram:: + + not_started -> running (first qualifying activity) + running -> offline (network unavailable) + offline -> running (heartbeat/upload recovers) + running -> suspended (IDE closes or sleeps) + suspended -> running (restart within resume grace; +1 generation) + suspended -> ended (resume grace expired) + running -> ended (explicit completion / idle timeout) + running -> revoked (withdrawal / consent pause) + offline -> revoked (server revocation observed) + ended, revoked -> (terminal) +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Callable, Optional + +from pydantic import ValidationError + +from .enums import ( + AgentRunOutcome, + CloseReason, + SessionReasonCode, + SessionState, +) +from .models import ( + AgentRunResult, + AgentRunV1, + ResearchSessionV1, + ResumeResult, + SessionIssue, + SessionPolicyV1, + SessionResult, + SessionTransition, +) + +if TYPE_CHECKING: + from research.participants.models import Enrollment + from typing import Any + +_LEGAL_TARGETS: dict[SessionState, frozenset[SessionState]] = { + SessionState.NOT_STARTED: frozenset({SessionState.RUNNING, SessionState.ENDED}), + SessionState.RUNNING: frozenset( + { + SessionState.OFFLINE, + SessionState.SUSPENDED, + SessionState.ENDED, + SessionState.REVOKED, + } + ), + SessionState.OFFLINE: frozenset( + {SessionState.RUNNING, SessionState.ENDED, SessionState.REVOKED} + ), + SessionState.SUSPENDED: frozenset({SessionState.RUNNING, SessionState.ENDED}), + SessionState.ENDED: frozenset(), + SessionState.REVOKED: frozenset(), +} + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def _issue(code: SessionReasonCode, message: str, field: str = "") -> SessionIssue: + return SessionIssue(code=code, message=message, field=field) + + +def _kill_switch_result( + session: ResearchSessionV1, kill_switch_check: Optional[Callable[[], bool]] +) -> Optional[SessionResult]: + """Return a non-accepted result when the injected kill switch is engaged. + + Returns ``None`` when no check is supplied or the switch is released, so the + caller proceeds with the pure state machine unchanged. + """ + if kill_switch_check is None or not kill_switch_check(): + return None + return SessionResult( + accepted=False, + session=session, + reason=SessionReasonCode.KILL_SWITCH_ENGAGED, + issue=_issue( + SessionReasonCode.KILL_SWITCH_ENGAGED, + "an operator kill switch is engaged for this scope", + "kill_switch", + ), + ) + + +def can_transition(from_state: SessionState, to_state: SessionState) -> bool: + """Whether ``to_state`` is a legal successor of ``from_state``.""" + return to_state in _LEGAL_TARGETS[from_state] + + +def policy_ref(policy: SessionPolicyV1) -> str: + """Human-readable, stable reference to the applied policy values.""" + return ( + f"idle_timeout_seconds={policy.idle_timeout_seconds}," + f"resume_grace_seconds={policy.resume_grace_seconds}" + ) + + +def session_policy_from_study(study: Any) -> Optional[SessionPolicyV1]: + """Extract a usable :class:`SessionPolicyV1` from a study, or ``None``. + + ``None`` means the study does not declare both the idle timeout and the + resume grace, so no timing decision can be made. The server never falls back + to a compiled default. A policy that fails validation is also ``None``: the + endpoints surface the typed ``POLICY_MISSING`` refusal instead of a 500. + """ + session_policy = (getattr(study, "research_config_json", None) or {}).get( + "session_policy", {} + ) + if ( + session_policy.get("idle_timeout_seconds") is None + or session_policy.get("resume_grace_seconds") is None + ): + return None + try: + return SessionPolicyV1( + idle_timeout_seconds=session_policy["idle_timeout_seconds"], + resume_grace_seconds=session_policy["resume_grace_seconds"], + heartbeat_seconds=session_policy.get("heartbeat_seconds"), + ) + except ValidationError: + return None + + +def open_session( + enrollment: Enrollment, + study: Any, + *, + manifest_digest: str, + environment_ref: Optional[str] = None, + context_id: str = "", + now: Optional[datetime] = None, +) -> ResearchSessionV1: + """Create a new, not-yet-started session bound to an enrollment/study.""" + return ResearchSessionV1( + research_session_id=uuid.uuid4(), + enrollment_id=enrollment.enrollment_id, + study_id=study.study_id, + context_id=context_id, + state=SessionState.NOT_STARTED, + opened_at=None, + last_activity_at=None, + closed_at=None, + close_reason=None, + resume_generation=0, + manifest_digest=manifest_digest, + environment_ref=environment_ref, + ) + + +def _apply( + session: ResearchSessionV1, + to_state: SessionState, + now: datetime, + *, + reason: SessionReasonCode = SessionReasonCode.OK, + close_reason: Optional[CloseReason] = None, + policy_reference: Optional[str] = None, + evidence_ref: Optional[str] = None, + resume_generation: Optional[int] = None, +) -> SessionResult: + if not can_transition(session.state, to_state): + return SessionResult( + accepted=False, + session=session, + reason=SessionReasonCode.INVALID_STATE_TRANSITION, + issue=_issue( + SessionReasonCode.INVALID_STATE_TRANSITION, + f"illegal session transition {session.state.value} -> {to_state.value}", + "state", + ), + ) + + updates: dict = {"state": to_state, "last_activity_at": now} + if to_state == SessionState.RUNNING: + updates["opened_at"] = session.opened_at or now + if to_state.is_terminal: + updates["closed_at"] = now + updates["close_reason"] = close_reason or ( + CloseReason.REVOKED + if to_state == SessionState.REVOKED + else CloseReason.EXPLICIT_COMPLETION + ) + if resume_generation is not None: + updates["resume_generation"] = resume_generation + + updated = session.model_copy(update=updates) + transition = SessionTransition( + transition_id=uuid.uuid4(), + research_session_id=session.research_session_id, + from_state=session.state, + to_state=to_state, + occurred_at=now, + reason=reason, + close_reason=updated.close_reason, + policy_ref=policy_reference, + evidence_ref=evidence_ref, + ) + return SessionResult( + accepted=True, session=updated, transition=transition, reason=reason + ) + + +def record_activity( + session: ResearchSessionV1, now: Optional[datetime] = None +) -> SessionResult: + """Update the last-activity marker without changing state.""" + timestamp = _now(now) + if session.state.is_terminal: + return SessionResult(accepted=True, session=session, reason=SessionReasonCode.OK) + return SessionResult( + accepted=True, + session=session.model_copy(update={"last_activity_at": timestamp}), + reason=SessionReasonCode.OK, + ) + + +def record_liveness( + session: ResearchSessionV1, now: Optional[datetime] = None +) -> SessionResult: + """Record a heartbeat as liveness only (ISSUE-06). + + A heartbeat proves the process is alive; it must never refresh + ``last_activity_at`` (that timestamp is the expiry/engagement authority) + or start a not-started session. Qualifying activity goes through + :func:`on_qualifying_activity` instead. + """ + timestamp = _now(now) + if session.state.is_terminal: + return SessionResult( + accepted=False, + session=session, + reason=SessionReasonCode.SESSION_TERMINAL, + issue=_issue( + SessionReasonCode.SESSION_TERMINAL, + "a terminal session cannot record liveness", + "state", + ), + ) + return SessionResult( + accepted=True, + session=session.model_copy(update={"last_heartbeat_at": timestamp}), + reason=SessionReasonCode.OK, + ) + + +def on_qualifying_activity( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + kill_switch_check: Optional[Callable[[], bool]] = None, + idle_timeout_seconds: Optional[int] = None, +) -> SessionResult: + """First qualifying activity: ``not_started -> running``. + + A not-started session is a placeholder: bootstrap stamps it when a window + opens or an idle rotation replaces the last session, and heartbeats keep it + alive while the IDE stays open. With ``idle_timeout_seconds``, a placeholder + that waited longer than that (a night, a lunch break) starts at this + activity rather than at its creation. + """ + timestamp = _now(now) + blocked = _kill_switch_result(session, kill_switch_check) + if blocked is not None: + return blocked + if session.state.is_terminal: + return SessionResult( + accepted=False, + session=session, + reason=SessionReasonCode.SESSION_TERMINAL, + issue=_issue( + SessionReasonCode.SESSION_TERMINAL, + "a terminal session cannot accept new activity", + "state", + ), + ) + if session.state == SessionState.NOT_STARTED: + waited = ( + idle_timeout_seconds is not None + and session.opened_at is not None + and (timestamp - session.opened_at).total_seconds() > idle_timeout_seconds + ) + start = session.model_copy(update={"opened_at": None}) if waited else session + return _apply(start, SessionState.RUNNING, timestamp) + return record_activity(session, timestamp) + + +def go_offline( + session: ResearchSessionV1, now: Optional[datetime] = None +) -> SessionResult: + """``running -> offline`` (network unavailable).""" + return _apply(session, SessionState.OFFLINE, _now(now)) + + +def recover( + session: ResearchSessionV1, now: Optional[datetime] = None +) -> SessionResult: + """``offline -> running`` (heartbeat/upload recovers).""" + return _apply(session, SessionState.RUNNING, _now(now)) + + +def suspend( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + evidence_ref: Optional[str] = None, +) -> SessionResult: + """``running -> suspended`` (IDE closes or the machine sleeps).""" + return _apply( + session, + SessionState.SUSPENDED, + _now(now), + evidence_ref=evidence_ref, + ) + + +def end( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + close_reason: CloseReason = CloseReason.EXPLICIT_COMPLETION, + evidence_ref: Optional[str] = None, +) -> SessionResult: + """End a live session (explicit completion or a caller-chosen reason).""" + timestamp = _now(now) + if session.state.is_terminal: + return SessionResult( + accepted=False, + session=session, + reason=SessionReasonCode.SESSION_TERMINAL, + issue=_issue( + SessionReasonCode.SESSION_TERMINAL, + f"session is already {session.state.value}", + "state", + ), + ) + return _apply( + session, + SessionState.ENDED, + timestamp, + close_reason=close_reason, + evidence_ref=evidence_ref, + ) + + +def revoke( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + evidence_ref: Optional[str] = None, +) -> SessionResult: + """Withdrawal/consent pause: terminal revocation.""" + timestamp = _now(now) + if session.state.is_terminal: + return SessionResult( + accepted=False, + session=session, + reason=SessionReasonCode.SESSION_TERMINAL, + issue=_issue( + SessionReasonCode.SESSION_TERMINAL, + f"session is already {session.state.value}", + "state", + ), + ) + return _apply( + session, + SessionState.REVOKED, + timestamp, + close_reason=CloseReason.REVOKED, + evidence_ref=evidence_ref, + ) + + +def resume( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + policy: SessionPolicyV1, +) -> ResumeResult: + """Reopen a suspended session within its resume grace, or end it once.""" + timestamp = _now(now) + + if session.state.is_terminal: + return ResumeResult( + accepted=False, + reopened=False, + requires_new_session=True, + session=session, + reason=SessionReasonCode.SESSION_TERMINAL, + issue=_issue( + SessionReasonCode.SESSION_TERMINAL, + f"session is already {session.state.value}", + "state", + ), + ) + if session.state != SessionState.SUSPENDED: + return ResumeResult( + accepted=False, + reopened=False, + requires_new_session=False, + session=session, + reason=SessionReasonCode.NOT_SUSPENDED, + issue=_issue( + SessionReasonCode.NOT_SUSPENDED, + f"session is {session.state.value}, not suspended", + "state", + ), + ) + + reference = policy_ref(policy) + last_activity = session.last_activity_at + within_grace = ( + last_activity is not None + and (timestamp - last_activity).total_seconds() <= policy.resume_grace_seconds + ) + + if within_grace: + result = _apply( + session, + SessionState.RUNNING, + timestamp, + policy_reference=reference, + resume_generation=session.resume_generation + 1, + ) + return ResumeResult( + accepted=result.accepted, + reopened=result.accepted, + requires_new_session=False, + session=result.session, + transition=result.transition, + reason=result.reason, + issue=result.issue, + ) + + ended = session.model_copy( + update={ + "state": SessionState.ENDED, + "closed_at": timestamp, + "close_reason": CloseReason.RESUME_GRACE_EXPIRED, + "last_activity_at": timestamp, + } + ) + transition = SessionTransition( + transition_id=uuid.uuid4(), + research_session_id=session.research_session_id, + from_state=session.state, + to_state=SessionState.ENDED, + occurred_at=timestamp, + reason=SessionReasonCode.RESUME_GRACE_EXPIRED, + close_reason=CloseReason.RESUME_GRACE_EXPIRED, + policy_ref=reference, + ) + return ResumeResult( + accepted=True, + reopened=False, + requires_new_session=True, + session=ended, + transition=transition, + reason=SessionReasonCode.RESUME_GRACE_EXPIRED, + ) + + +def expire_if_idle( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + policy: SessionPolicyV1, + kill_switch_check: Optional[Callable[[], bool]] = None, +) -> SessionResult: + """End a session silent past the idle timeout: a live one on its last + qualifying activity, a not-started one on its last heartbeat.""" + timestamp = _now(now) + blocked = _kill_switch_result(session, kill_switch_check) + if blocked is not None: + return blocked + if session.state.is_terminal: + return SessionResult(accepted=True, session=session, reason=SessionReasonCode.OK) + # A window that never started its session keeps it across restarts, days + # apart: its interval would then run from the first start to the latest + # heartbeat. Once it has been silent past the timeout it ends, and the + # window's next start gets a fresh session. + last_activity = ( + session.last_heartbeat_at or session.opened_at + if session.state == SessionState.NOT_STARTED + else session.last_activity_at + ) + if last_activity is None: + return SessionResult(accepted=True, session=session, reason=SessionReasonCode.OK) + if (timestamp - last_activity).total_seconds() <= policy.idle_timeout_seconds: + return SessionResult(accepted=True, session=session, reason=SessionReasonCode.OK) + return _apply( + session, + SessionState.ENDED, + timestamp, + reason=SessionReasonCode.IDLE_TIMEOUT, + close_reason=CloseReason.IDLE_TIMEOUT, + policy_reference=policy_ref(policy), + ) + + +def close( + session: ResearchSessionV1, + reason: CloseReason, + now: Optional[datetime] = None, + *, + kill_switch_check: Optional[Callable[[], bool]] = None, +) -> SessionResult: + """Apply a typed close reason to a session.""" + timestamp = _now(now) + blocked = _kill_switch_result(session, kill_switch_check) + if blocked is not None: + return blocked + if reason == CloseReason.REVOKED: + return revoke(session, timestamp, evidence_ref=f"close_reason={reason.value}") + if reason == CloseReason.IDLE_TIMEOUT: + return _apply( + session, + SessionState.ENDED, + timestamp, + reason=SessionReasonCode.IDLE_TIMEOUT, + close_reason=CloseReason.IDLE_TIMEOUT, + evidence_ref=f"close_reason={reason.value}", + ) + if reason == CloseReason.RESUME_GRACE_EXPIRED: + return _apply( + session, + SessionState.ENDED, + timestamp, + reason=SessionReasonCode.RESUME_GRACE_EXPIRED, + close_reason=CloseReason.RESUME_GRACE_EXPIRED, + evidence_ref=f"close_reason={reason.value}", + ) + return end( + session, + timestamp, + close_reason=reason, + evidence_ref=f"close_reason={reason.value}", + ) + + +def start_agent_run( + session: ResearchSessionV1, + now: Optional[datetime] = None, + *, + agent_release_id: Optional[str] = None, +) -> AgentRunResult: + """Start an agent run; allowed only while the session is ``RUNNING``.""" + timestamp = _now(now) + if session.state != SessionState.RUNNING: + return AgentRunResult( + accepted=False, + session=session, + reason=SessionReasonCode.SESSION_NOT_RUNNING, + issue=_issue( + SessionReasonCode.SESSION_NOT_RUNNING, + f"an agent run requires a RUNNING session, not {session.state.value}", + "state", + ), + ) + run = AgentRunV1( + agent_run_id=uuid.uuid4(), + research_session_id=session.research_session_id, + agent_release_id=agent_release_id, + started_at=timestamp, + ) + return AgentRunResult( + accepted=True, run=run, session=session, reason=SessionReasonCode.OK + ) + + +def end_agent_run( + run: AgentRunV1, + outcome: AgentRunOutcome, + now: Optional[datetime] = None, +) -> AgentRunResult: + """End an agent run; idempotent once terminal.""" + timestamp = _now(now) + if run.is_terminal: + return AgentRunResult( + accepted=True, run=run, session=None, reason=SessionReasonCode.AGENT_RUN_TERMINAL + ) + ended = run.model_copy(update={"ended_at": timestamp, "outcome": outcome}) + return AgentRunResult( + accepted=True, run=ended, session=None, reason=SessionReasonCode.OK + ) + + +def on_agent_run_crashed( + run: AgentRunV1, + session: ResearchSessionV1, + now: Optional[datetime] = None, +) -> AgentRunResult: + """Crash handling: end the run; the research session is preserved.""" + result = end_agent_run(run, AgentRunOutcome.CRASHED, now) + return AgentRunResult( + accepted=result.accepted, + run=result.run, + session=session, + reason=result.reason, + issue=result.issue, + ) diff --git a/src/research/runtime/sessions/store.py b/src/research/runtime/sessions/store.py new file mode 100644 index 0000000..6db3692 --- /dev/null +++ b/src/research/runtime/sessions/store.py @@ -0,0 +1,385 @@ +"""CRUD-style persistence helpers for research sessions and agent runs. + +These functions take a caller-managed SQLAlchemy ``Session`` so the core package +never imports ``App`` or touches the application singleton. Session and agent-run +rows are distinct tables/identities; the store never treats one id as the other. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Sequence + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError + +if TYPE_CHECKING: + import uuid + + from sqlalchemy.orm import Session + + from .models import ( + AgentRunV1, + ResearchSessionV1, + SessionTransition, + ) + +from database.research_schemas import ResearchAgentRun +from database.research_schemas import ResearchSessionV1 as ResearchSessionRow + +from .enums import AgentRunOutcome, CloseReason, SessionState + +_TERMINAL_STATES = (SessionState.ENDED.value, SessionState.REVOKED.value) + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _session_row(research_session: ResearchSessionV1) -> ResearchSessionRow: + return ResearchSessionRow( + session_id=research_session.research_session_id, + enrollment_id=research_session.enrollment_id, + study_id=research_session.study_id, + context_id=research_session.context_id or str(research_session.research_session_id), + state=research_session.state.value, + opened_at=research_session.opened_at, + last_heartbeat_at=research_session.last_heartbeat_at, + last_activity_at=research_session.last_activity_at, + closed_at=research_session.closed_at, + close_reason=( + research_session.close_reason.value + if research_session.close_reason is not None + else None + ), + resume_generation=research_session.resume_generation, + manifest_digest=research_session.manifest_digest, + environment_json={ + "environment_ref": research_session.environment_ref + }, + created_at=_now(), + ) + + +def create_session( + session: Session, + research_session: ResearchSessionV1, + *, + commit: bool = True, +) -> ResearchSessionRow: + """Insert a new research session row, atomically. + + The partial unique index ``uq_research_session_active_context`` allows at + most one non-terminal session per ``(enrollment_id, context_id)``. A savepoint + around the insert means a concurrent create that loses the race rolls back + only its own insert and returns the winning session for that same context + instead of leaving a duplicate (or aborting the caller's transaction). + + ``commit=False`` lets bootstrap own one unit of work (assignment + session + commit together, or neither). + """ + row = _session_row(research_session) + try: + with session.begin_nested(): + session.add(row) + except IntegrityError: + existing = get_active_session_for_context( + session, research_session.enrollment_id, row.context_id + ) + if existing is not None: + return existing + raise + if commit: + session.commit() + else: + session.flush() + session.refresh(row) + return row + + +def get_session( + session: Session, research_session_id: uuid.UUID, *, for_update: bool = False +) -> Optional[ResearchSessionRow]: + """Fetch a session row by id, or ``None``. + + ``for_update`` takes a row lock (``SELECT ... FOR UPDATE``) so ingestion can + serialize concurrent batches for the same session. + """ + if not for_update: + return session.get(ResearchSessionRow, research_session_id) + statement = ( + select(ResearchSessionRow) + .where(ResearchSessionRow.session_id == research_session_id) + .with_for_update() + ) + return session.execute(statement).scalars().first() + + +def list_sessions( + session: Session, enrollment_id: Optional[uuid.UUID] = None +) -> Sequence[ResearchSessionRow]: + """List sessions, optionally for one enrollment, newest-first.""" + statement = select(ResearchSessionRow) + if enrollment_id is not None: + statement = statement.where(ResearchSessionRow.enrollment_id == enrollment_id) + statement = statement.order_by(ResearchSessionRow.created_at.desc()) + return list(session.execute(statement).scalars().all()) + + +def get_active_session_for_enrollment( + session: Session, enrollment_id: uuid.UUID +) -> Optional[ResearchSessionRow]: + """Return the newest non-terminal session for an enrollment, or ``None``.""" + statement = ( + select(ResearchSessionRow) + .where( + ResearchSessionRow.enrollment_id == enrollment_id, + ResearchSessionRow.state.notin_(_TERMINAL_STATES), + ) + .order_by(ResearchSessionRow.created_at.desc()) + ) + return session.execute(statement).scalars().first() + + +def get_active_session_for_context( + session: Session, + enrollment_id: uuid.UUID, + context_id: str, + *, + for_update: bool = False, +) -> Optional[ResearchSessionRow]: + """Return the non-terminal session for ``(enrollment, context)``, or ``None``. + + This is the idempotent-creation lookup: one authenticated execution context + maps to at most one live session. ``for_update`` locks the row so a caller + that may close it (idle expiry) serializes with concurrent heartbeats. + """ + statement = ( + select(ResearchSessionRow) + .where( + ResearchSessionRow.enrollment_id == enrollment_id, + ResearchSessionRow.context_id == context_id, + ResearchSessionRow.state.notin_(_TERMINAL_STATES), + ) + .order_by(ResearchSessionRow.created_at.desc()) + ) + if for_update: + statement = statement.with_for_update() + return session.execute(statement).scalars().first() + + +def update_session( + session: Session, research_session: ResearchSessionV1, *, commit: bool = True +) -> Optional[ResearchSessionRow]: + """Persist a state/timestamp/close change onto an existing session row. + + ``commit=False`` stages the change for a caller-owned unit of work. + """ + row = session.get(ResearchSessionRow, research_session.research_session_id) + if row is None: + return None + row.state = research_session.state.value + row.opened_at = research_session.opened_at + row.last_heartbeat_at = research_session.last_heartbeat_at + row.last_activity_at = research_session.last_activity_at + row.closed_at = research_session.closed_at + row.close_reason = ( + research_session.close_reason.value + if research_session.close_reason is not None + else None + ) + row.resume_generation = research_session.resume_generation + row.manifest_digest = research_session.manifest_digest + row.environment_json = {"environment_ref": research_session.environment_ref} + if commit: + session.commit() + session.refresh(row) + else: + session.flush() + return row + + +def insert_transition( + session: Session, transition: SessionTransition, *, commit: bool = True +) -> SessionTransition: + """Append one recorded state transition to its session's log. + + The transition log is ``research_session.transitions_json`` (the session is + the only owner of its transitions), so no child row is written. + ``commit=False`` stages the change for a caller-owned unit of work. + """ + row = session.get(ResearchSessionRow, transition.research_session_id) + if row is None: # pragma: no cover - guarded by the session router + raise ValueError(f"research session {transition.research_session_id} not found") + existing = row.transitions_json + serialized = list(existing) if isinstance(existing, list) else [] + serialized.append(transition.model_dump(mode="json")) + row.transitions_json = serialized + session.add(row) + if commit: + session.commit() + session.refresh(row) + else: + session.flush() + return transition + + +def resolve_open_context_session( + session: Session, + enrollment_id: uuid.UUID, + context_id: str, + *, + study: Any, + now: datetime, + commit: bool = True, + kill_switch_check: Optional[Callable[[], bool]] = None, +) -> Optional[ResearchSessionRow]: + """The live session for ``(enrollment, context)``, expiring an idle one first. + + Bootstrap and session create used to hand a returning client its previous + context session even when that session had already exceeded the study's + idle timeout; the first heartbeat then ended it and the client had to + activate a third time (review D-01/C-02). Here an idle-expired session is + closed (``IDLE_TIMEOUT`` transition, persisted) and ``None`` is returned so + the caller opens a fresh one. Without a usable study policy the session is + returned as is (the server never invents a timeout). The row is locked + while it is inspected, and an engaged kill switch (``kill_switch_check``) + leaves it untouched exactly like the heartbeat path does. + """ + from .service import expire_if_idle, session_policy_from_study + + row = get_active_session_for_context(session, enrollment_id, context_id, for_update=True) + if row is None: + return None + policy = session_policy_from_study(study) + if policy is None: + return row + model = row_to_session(row) + idle = expire_if_idle(model, now, policy=policy, kill_switch_check=kill_switch_check) + if not idle.accepted or idle.transition is None: + return row + insert_transition(session, idle.transition, commit=commit) + update_session(session, idle.session, commit=commit) + return None + + +def create_agent_run(session: Session, run: AgentRunV1) -> ResearchAgentRun: + """Insert a new agent-run row (distinct id from its session).""" + row = ResearchAgentRun( + agent_run_id=run.agent_run_id, + research_session_id=run.research_session_id, + agent_release_id=run.agent_release_id, + assignment_id=run.assignment_id, + agent_profile_id=run.agent_profile_id, + profile_digest=run.profile_digest, + profile_snapshot_json=run.profile_snapshot_json, + started_at=run.started_at, + ended_at=run.ended_at, + outcome=run.outcome.value if run.outcome is not None else None, + ) + session.add(row) + session.commit() + session.refresh(row) + return row + + +def get_agent_run( + session: Session, agent_run_id: uuid.UUID +) -> Optional[ResearchAgentRun]: + """Fetch an agent-run row by id, or ``None``.""" + return session.get(ResearchAgentRun, agent_run_id) + + +def list_agent_runs( + session: Session, research_session_id: uuid.UUID +) -> Sequence[ResearchAgentRun]: + """List the agent runs of one session, oldest-first.""" + statement = ( + select(ResearchAgentRun) + .where(ResearchAgentRun.research_session_id == research_session_id) + .order_by(ResearchAgentRun.started_at.asc()) + ) + return list(session.execute(statement).scalars().all()) + + +def row_to_session(row: ResearchSessionRow) -> ResearchSessionV1: + """Rehydrate a session row into the domain model.""" + from .models import ResearchSessionV1 as ResearchSessionModel + + state = row.state + close_reason = row.close_reason + environment = row.environment_json or {} + return ResearchSessionModel( + research_session_id=row.session_id, + enrollment_id=row.enrollment_id, + study_id=row.study_id, + context_id=row.context_id or "", + state=state if isinstance(state, SessionState) else SessionState(state), + opened_at=row.opened_at, + last_heartbeat_at=row.last_heartbeat_at, + last_activity_at=row.last_activity_at, + closed_at=row.closed_at, + close_reason=( + close_reason + if isinstance(close_reason, CloseReason) or close_reason is None + else CloseReason(close_reason) + ), + resume_generation=row.resume_generation, + manifest_digest=row.manifest_digest, + environment_ref=environment.get("environment_ref"), + ) + + +def row_to_agent_run(row: ResearchAgentRun) -> AgentRunV1: + """Rehydrate an agent-run row into the domain model.""" + from .models import AgentRunV1 as AgentRunModel + + outcome = row.outcome + return AgentRunModel( + agent_run_id=row.agent_run_id, + research_session_id=row.research_session_id, + agent_release_id=row.agent_release_id, + assignment_id=row.assignment_id, + agent_profile_id=row.agent_profile_id, + profile_digest=row.profile_digest, + profile_snapshot_json=row.profile_snapshot_json, + started_at=row.started_at, + ended_at=row.ended_at, + outcome=( + outcome + if isinstance(outcome, AgentRunOutcome) or outcome is None + else AgentRunOutcome(outcome) + ), + ) + + +def row_to_transition(row: Mapping[str, Any]) -> SessionTransition: + """Rehydrate one serialized ``transitions_json`` entry.""" + from .models import SessionTransition as SessionTransitionModel + + return SessionTransitionModel.model_validate(dict(row)) + + +def session_summary(row: ResearchSessionRow) -> dict[str, Any]: + """Return a compact, non-secret session summary safe for responses.""" + environment = row.environment_json or {} + return { + "research_session_id": str(row.session_id), + "enrollment_id": str(row.enrollment_id), + "study_id": str(row.study_id), + "context_id": row.context_id, + "state": row.state, + "opened_at": _iso(row.opened_at), + # Liveness and qualifying activity are separate facts (ISSUE-06). + "last_heartbeat_at": _iso(row.last_heartbeat_at), + "last_activity_at": _iso(row.last_activity_at), + "closed_at": _iso(row.closed_at), + "close_reason": row.close_reason, + "resume_generation": row.resume_generation, + "manifest_digest": row.manifest_digest, + "environment_ref": environment.get("environment_ref"), + } + + +def _iso(value: Optional[datetime]) -> Optional[str]: + return value.isoformat() if isinstance(value, datetime) else None diff --git a/src/research/study/__init__.py b/src/research/study/__init__.py new file mode 100644 index 0000000..d6e7c5d --- /dev/null +++ b/src/research/study/__init__.py @@ -0,0 +1,13 @@ +"""Study definition, agent registry and runtime packaging. + +Merged home for the versioned study protocol/publication service, the agent +release registry and capability snapshots, and the runtime packaging and +conformance suite. The individual sub-packages keep their original public +surfaces: + +* :mod:`research.study.protocol` +* :mod:`research.study.agents` +* :mod:`research.study.packaging` + +The core packages never import ``App``, FastAPI, or a session factory. +""" diff --git a/src/research/study/agents/__init__.py b/src/research/study/agents/__init__.py new file mode 100644 index 0000000..87d8bf1 --- /dev/null +++ b/src/research/study/agents/__init__.py @@ -0,0 +1,86 @@ +"""Agent registry and capability contract (Issue 04). + +Public surface: + +* :mod:`research.study.agents.enums` - closed vocabularies for qualification status, + distribution source, snapshot capability state, coverage state, registry + reason codes, plus the re-exported Issue 01 ``Fidelity``. +* :mod:`research.study.agents.models` - Pydantic v2 contracts for + :class:`~research.study.agents.models.AgentReleaseV1` (distribution + adapter + identity) and :class:`~research.study.agents.models.CapabilitySnapshotV1` (separate + declared/observed maps with explicit ``UNKNOWN``/``UNAVAILABLE``). +* :mod:`research.study.agents.registry` - the pure :class:`~research.study.agents.registry.AgentRegistry` + service: digest-identity registration, qualification transitions, exact + platform resolution, qualification assessment and capability coverage. +* :mod:`research.study.agents.resolver` - :class:`~research.study.agents.resolver.RegistryReleaseResolver`, + the bridge implementing :class:`research.study.protocol.validation.ReleaseResolver` + so protocol publication rejects missing/unqualified/withdrawn releases. + +The persistence adapters live in :mod:`research.study.agents.store` and take a +caller-supplied SQLAlchemy ``Session`` so this package stays free of any +application/App import. +""" + +from .enums import ( + CapabilityCoverageState, + DistributionSourceType, + Fidelity, + QualificationStatus, + RegistryReasonCode, + SnapshotCapabilityState, +) +from .models import ( + AdapterRef, + AgentReleaseV1, + CapabilityCoverage, + CapabilityCoverageReport, + CapabilityEntry, + CapabilitySnapshotV1, + DistributionArtifact, + EnvironmentRef, + ReleaseDisplay, + ReleaseTests, + SnapshotFailure, +) +from .registry import ( + AgentRegistry, + ArtifactResolution, + QualificationAssessment, + RegistrationResult, + RegistryIssue, + build_capability_snapshot, + capability_coverage, + coverage_report, + derive_qualification_status, +) +from .resolver import RegistryReleaseResolver + +__all__ = [ + "AdapterRef", + "AgentRegistry", + "AgentReleaseV1", + "ReleaseTests", + "ArtifactResolution", + "CapabilityCoverage", + "CapabilityCoverageReport", + "CapabilityCoverageState", + "CapabilityEntry", + "CapabilitySnapshotV1", + "DistributionArtifact", + "DistributionSourceType", + "EnvironmentRef", + "Fidelity", + "QualificationAssessment", + "QualificationStatus", + "RegistrationResult", + "RegistryIssue", + "RegistryReasonCode", + "RegistryReleaseResolver", + "ReleaseDisplay", + "SnapshotCapabilityState", + "SnapshotFailure", + "build_capability_snapshot", + "capability_coverage", + "coverage_report", + "derive_qualification_status", +] diff --git a/src/research/study/agents/distributions.py b/src/research/study/agents/distributions.py new file mode 100644 index 0000000..c7e28c2 --- /dev/null +++ b/src/research/study/agents/distributions.py @@ -0,0 +1,812 @@ +"""Resolve an ``AgentProfile`` *distribution* into a non-secret view. + +An :class:`~database.db_schemas.AgentProfile` IS a distribution: it pins the +exact agent release a study condition runs. This module turns a profile row plus +its (optional) registry release into the +:class:`~research.study.protocol.validation.DistributionResolution` the protocol +validator consumes, and into the derived, read-only shape the admin UI needs. + +A profile row owns only the pin (``profile_id``/``release_id``) and the +executable config. Distribution mode, command/package identity and argument +vector are derived **exclusively from the release** — the profile-level +``distribution_mode``/``agent_package``/``agent_command``/``agent_command_args`` +attributes no longer exist and are never read. + +The module also owns the shared, pure profile↔release executable contract +(:func:`validate_profile_configuration`) used by profile CRUD and study +creation, so an unexecutable combination is rejected before enrollment. + +Verification is **derived**, never stored: + +* ``PACKAGED`` is verified only when its release is ``QUALIFIED`` (at least one + administrator-approved host platform). +* ``BYOA_EXTERNAL`` is *always* unverified: a participant-installed agent has no + artifact to bind an approval to. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from typing import TYPE_CHECKING, Any, Mapping, Optional + +from agents.tools import ( + HARNESS_PROFILE_FIELDS, + tools_for_framework, + validate_command_timeout_seconds, + validate_commands_denylist, + validate_harness_options, +) +from research.study.protocol.enums import ReleaseResolutionStatus +from research.study.protocol.models import ResolvedAgentConfig +from research.study.protocol.validation import DistributionResolution + +from .enums import ( + INFERENCE_GATEWAY_FRAMEWORKS, + MANAGED_RUNTIME_FRAMEWORK, + DistributionMode, + QualificationStatus, +) +from .models import ( + BYOA_BINDING_FIELDS, + BYOA_CONFIG_FIELDS, + BYOA_CONFIG_TRANSPORTS, + BYOA_CREDENTIAL_FIELD, + BYOA_PROVIDER_KIND_FIELD, + BYOA_PROVIDER_KIND_OPENAI_COMPATIBLE, + BYOA_RUNTIME_FIELDS, +) +from .registry import ( + ALL_APPROVAL_OPTIONS, + AgentRegistry, +) +from .resolver import RegistryReleaseResolver + +if TYPE_CHECKING: + from .models import AgentReleaseV1 + +__all__ = [ + "BYOA_ALWAYS_SET_FIELDS", + "FRAMEWORK_DISTRIBUTION_MODES", + "PACKAGED_CONFIGURABLE_FIELDS", + "ProfileConfigurationError", + "distribution_supported_platforms", + "parse_command_args", + "release_profile_configurability", + "missing_gateway_bindings", + "release_bindings", + "requires_inference_gateway", + "resolve_distribution_view", + "validate_profile_configuration", +] + +_WITHDRAWN = frozenset( + {QualificationStatus.RETIRED, QualificationStatus.BLOCKED} +) + +#: The distribution mode each supported framework can execute (ISSUE-03). +#: +#: The managed built-in runtime is digest-pinned (``PACKAGED``); Goose and Codex +#: are participant-installed BYOA identities. Framework and release are +#: validated together so an arm's label always matches what will run. +FRAMEWORK_DISTRIBUTION_MODES: dict[str, str] = { + MANAGED_RUNTIME_FRAMEWORK: DistributionMode.PACKAGED.value, + "goose": DistributionMode.BYOA_EXTERNAL.value, + "codex": DistributionMode.BYOA_EXTERNAL.value, +} + +#: Profile fields that govern the managed (``PACKAGED``) runtime: all of them +#: reach it through the managed policy, including the per-turn context window +#: and the researcher-authored system prompt. +PACKAGED_CONFIGURABLE_FIELDS: tuple[str, ...] = ( + "model", + "temperature", + "max_steps", + "tools", + "approval_policy", + "max_context_tokens", + "system_prompt", +) + +#: Profile fields every profile sets (``model``/``max_steps``/``approval_policy`` +#: are required on the profile), so a BYOA release that does not bind one of them +#: can never be pinned: :func:`validate_profile_configuration` rejects it as +#: ``BYOA_CONFIG_UNMAPPED``. +BYOA_ALWAYS_SET_FIELDS: tuple[str, ...] = ("model", "max_steps", "approval_policy") + +#: Launcher suffixes stripped from a BYOA command before it is compared with a +#: framework name (``codex.exe`` declares ``codex``). +_COMMAND_SUFFIXES = (".exe", ".cmd", ".bat") + + +class ProfileConfigurationError(ValueError): + """Typed profile/release executable-contract violation. + + ``str(error)`` is ``"CODE: message"`` (the convention the study router + already parses) and ``code`` is the machine-readable reason. + """ + + def __init__(self, code: str, message: str, field: str = ""): + super().__init__(f"{code}: {message}") + self.code = code + self.field = field + + +def _release_document( + release: Any, release_json: Optional[Mapping[str, Any]] +) -> Optional[Mapping[str, Any]]: + """The raw evidence document for a release model, row or mapping.""" + if isinstance(release_json, Mapping): + return release_json + candidate = getattr(release, "release_json", None) + if isinstance(candidate, Mapping): + return candidate + if isinstance(release, Mapping): + return release + return None + + +def _framework(value: Any) -> str: + return str(value or "").strip().lower() + + +def _release_mode(release: Any) -> str: + mode = getattr(release, "distribution_mode", None) + value = getattr(mode, "value", mode) + return str(value or "").strip().upper() + + +def _qualification_value(release: Any, document: Optional[Mapping[str, Any]]) -> str: + status = getattr(release, "qualification_status", None) + value = str(getattr(status, "value", status) or "").strip().upper() + if value: + return value + return QualificationStatus.UNQUALIFIED.value + + +def _profile_tools(profile: Any) -> set[str]: + raw = getattr(profile, "tools_json", None) + if isinstance(raw, (list, tuple)): + return {str(item) for item in raw} + try: + parsed = json.loads(raw or "[]") + except (TypeError, ValueError) as error: + raise ProfileConfigurationError( + "TOOLS_INVALID", + "tools_json must be a JSON array of tool names", + "tools_json", + ) from error + if not isinstance(parsed, list) or not all(isinstance(item, str) for item in parsed): + raise ProfileConfigurationError( + "TOOLS_INVALID", + "tools_json must be a JSON array of tool names", + "tools_json", + ) + return set(parsed) + + +def _byoa_bindings(release: Any, document: Optional[Mapping[str, Any]]) -> dict[str, Mapping[str, Any]]: + """The declared field→transport bindings of a BYOA release, keyed by field.""" + raw: Any = None + if release is not None and getattr(release, "byoa_config", None): + raw = [] + for binding in release.byoa_config: + raw.append( + binding.model_dump(mode="json") + if hasattr(binding, "model_dump") + else binding + ) + elif document is not None: + raw = document.get("byoa_config") + bindings: dict[str, Mapping[str, Any]] = {} + if isinstance(raw, (list, tuple)): + for item in raw: + if not isinstance(item, Mapping): + continue + field = str(item.get("field") or "").strip().lower() + transport = str(item.get("transport") or "").strip().lower() + key = str(item.get("key") or "").strip() + if field in BYOA_BINDING_FIELDS and transport in BYOA_CONFIG_TRANSPORTS and key: + bindings[field] = item + return bindings + + +def release_bindings(release: Any, document: Optional[Mapping[str, Any]] = None) -> dict[str, Mapping[str, Any]]: + """Public accessor for a release's declared bindings keyed by field.""" + return _byoa_bindings(release, document) + + +def requires_inference_gateway(framework: Optional[str]) -> bool: + """Whether profiles of ``framework`` call the research inference gateway.""" + return str(framework or "").strip().lower() in INFERENCE_GATEWAY_FRAMEWORKS + + +def _uniquely_gateway_bound(release: Any, document: Optional[Mapping[str, Any]]) -> bool: + """Whether the release provably launches a gateway-bound agent. + + Its declared identity must name exactly one framework, a gateway-bound one, + and the executable it launches (the command, else the package) must be that + framework's own: only a launched Goose calls the research gateway, so a + ``codex-acp`` command under a Goose label proves nothing. + """ + tokens = _declared_identity_tokens(release, document) + declared = [ + framework + for framework, mode in FRAMEWORK_DISTRIBUTION_MODES.items() + if mode == DistributionMode.BYOA_EXTERNAL.value and framework in tokens + ] + if len(declared) != 1 or not requires_inference_gateway(declared[0]): + return False + launched = None + for attribute in ("agent_command", "agent_package"): + value = getattr(release, attribute, None) + if value is None and document is not None: + value = document.get(attribute) + if str(value or "").strip(): + launched = _declared_identity_tokens(SimpleNamespace(agent_command=value), None) if attribute == "agent_command" else {str(value).strip().lower()} + break + return launched == {declared[0]} + + +def _is_gateway_tools_binding(binding: Optional[Mapping[str, Any]]) -> bool: + return binding is not None and str(binding.get("transport") or "").strip().lower() == "gateway" + + +def release_enforces_tools_at_gateway( + release: Any, release_json: Optional[Mapping[str, Any]] = None +) -> bool: + """Whether the release declares its ``tools`` binding with the ``gateway`` transport. + + The research inference gateway then offers the model only the frozen + profile's tool selection (an empty selection means no tools); otherwise it + passes the agent's tool definitions through untouched. + """ + bindings = _byoa_bindings(release, _release_document(release, release_json)) + return _is_gateway_tools_binding(bindings.get("tools")) + + +def missing_gateway_bindings(bindings: Mapping[str, Mapping[str, Any]]) -> list[str]: + """The runtime bindings a gateway-bound release lacks (empty when complete). + + Every field in ``BYOA_RUNTIME_FIELDS`` must be bound with the ``env`` + transport (the plugin fills them at launch; the credential must never be + an argv token), and the ``provider_kind`` binding must translate the + server's ``openai_compatible`` value into the agent's own vocabulary. + """ + missing: list[str] = [] + for field in BYOA_RUNTIME_FIELDS: + binding = bindings.get(field) + if binding is None: + missing.append(field) + continue + transport = str(binding.get("transport") or "").strip().lower() + if transport != "env": + missing.append(f"{field} (env transport required)") + continue + if field == BYOA_PROVIDER_KIND_FIELD: + value_map = binding.get("value_map") or {} + if not isinstance(value_map, Mapping) or not str( + value_map.get(BYOA_PROVIDER_KIND_OPENAI_COMPATIBLE) or "" + ).strip(): + missing.append( + f"{field} (value_map must translate {BYOA_PROVIDER_KIND_OPENAI_COMPATIBLE!r})" + ) + return missing + + +def validate_profile_configuration( + profile: Any, + release: Optional[AgentReleaseV1], + *, + release_json: Optional[Mapping[str, Any]] = None, +) -> None: + """Reject a profile/release combination that cannot execute (ISSUE-03). + + One shared invariant for profile create/update and the study-creation freeze: + + * the profile must pin a registered release; + * ``code4me2-agent`` requires a ``PACKAGED`` release; ``goose``/``codex`` + require a ``BYOA_EXTERNAL`` release; + * a BYOA release must declare a command/package identity; + * the selected release must be qualified (never withdrawn); + * selected tools must belong to the framework's catalogue; + * for BYOA, every field the profile sets must be covered by a declared + configuration binding; + * for BYOA, ``max_context_tokens`` and ``system_prompt`` must stay unset: no + binding can forward them to an externally installed agent, so they would + only be labels; the same holds for the built-in runtime's command and + harness settings (``commands_denylist`` / ``command_timeout_seconds`` / + ``harness_options``, decision D-01); + * those settings must be valid, and a ``harness_options.verify_command`` + must not run a program the profile's own ``commands_denylist`` blocks. + + Raises :class:`ProfileConfigurationError` (a ``ValueError`` whose ``str`` is + ``"CODE: message"``). Nothing is mutated and no database session is needed. + """ + framework = _framework(getattr(profile, "framework_version", None)) + document = _release_document(release, release_json) + + if release is None: + raise ProfileConfigurationError( + "RELEASE_UNRESOLVED", + "the profile pins no registered release to execute", + "release_id", + ) + + expected_mode = FRAMEWORK_DISTRIBUTION_MODES.get(framework) + if expected_mode is None: + raise ProfileConfigurationError( + "FRAMEWORK_UNSUPPORTED", + "framework_version must be one of " + + ", ".join(sorted(FRAMEWORK_DISTRIBUTION_MODES)), + "framework_version", + ) + + qualification = _qualification_value(release, document) + if qualification in {status.value for status in _WITHDRAWN}: + raise ProfileConfigurationError( + "RELEASE_WITHDRAWN", + f"release {getattr(release, 'release_id', '')!r} is withdrawn", + "release_id", + ) + if qualification != QualificationStatus.QUALIFIED.value: + raise ProfileConfigurationError( + "RELEASE_NOT_QUALIFIED", + f"release {getattr(release, 'release_id', '')!r} is not qualified", + "release_id", + ) + + mode = _document_mode(release, document) + if mode != expected_mode: + raise ProfileConfigurationError( + "FRAMEWORK_DISTRIBUTION_MISMATCH", + f"framework {framework!r} cannot execute a {mode} release; " + f"it requires {expected_mode}", + "release_id", + ) + + if mode == DistributionMode.BYOA_EXTERNAL.value: + identity = getattr(release, "byoa_identity", None) + if identity is None and document is not None: + identity = ( + str(document.get("agent_command") or "").strip() + or str(document.get("agent_package") or "").strip() + or None + ) + if not identity: + raise ProfileConfigurationError( + "DISTRIBUTION_IDENTITY_MISSING", + "a BYOA release must declare an agent_command or agent_package identity", + "release_id", + ) + + # ``max_context_tokens`` has no BYOA binding vocabulary at all: only the + # managed runtime receives it (through its managed policy); it is never + # forwarded to an externally installed agent. Refuse it instead of + # storing a label that does not govern the external process. + if getattr(profile, "max_context_tokens", None) is not None: + raise ProfileConfigurationError( + "BYOA_FIELD_UNSUPPORTED", + "max_context_tokens is not forwarded to externally installed " + "agents (goose/codex); leave it empty for a BYOA runtime", + "max_context_tokens", + ) + # The researcher-authored system prompt likewise only reaches the + # managed runtime (agent-config + run policy); an external agent keeps + # its own prompt, so a stored value would be a label, not a condition. + if getattr(profile, "system_prompt", None) is not None: + raise ProfileConfigurationError( + "BYOA_FIELD_UNSUPPORTED", + "system_prompt is not forwarded to externally installed " + "agents (goose/codex); leave it empty for a BYOA runtime", + "system_prompt", + ) + # The built-in runtime's blocked commands, command timeout and + # harness switches (D-01) only reach the managed runtime through its + # run policy; an external agent enforces its own. + for field in HARNESS_PROFILE_FIELDS: + if getattr(profile, field, None) is not None: + raise ProfileConfigurationError( + "BYOA_FIELD_UNSUPPORTED", + f"{field} is not forwarded to externally installed agents " + "(goose/codex); leave it empty for a BYOA runtime", + field, + ) + + # ISSUE-03 Path A: every frozen profile field the profile actually sets + # must be covered by a declared translation. A field with no binding is + # refused here, before enrollment, instead of becoming an experimental + # label that does not govern the external process. The provider + # connection itself is never translated: a gateway-bound agent (Goose) + # receives the research inference gateway address plus a per-participant + # capability through the runtime bindings below, and the server-held + # key stays on the server. + bindings = _byoa_bindings(release, document) + tools = _profile_tools(profile) + required_fields = { + "model": bool(str(getattr(profile, "model", "") or "").strip()), + "temperature": getattr(profile, "temperature", None) is not None, + "max_steps": getattr(profile, "max_steps", None) is not None, + "tools": bool(tools), + "approval_policy": bool( + str(getattr(profile, "approval_policy", "") or "").strip() + ), + } + unmapped = sorted( + field for field, needed in required_fields.items() if needed and field not in bindings + ) + if unmapped: + raise ProfileConfigurationError( + "BYOA_CONFIG_UNMAPPED", + "the release declares no configuration translation for " + + ", ".join(unmapped) + + "; the profile's values would not govern the external agent", + "release_id", + ) + # A gateway-bound framework (Goose) must be able to reach the research + # inference gateway: without the runtime bindings the participant's own + # provider configuration would be used, unmetered. Fail closed. + if requires_inference_gateway(getattr(profile, "framework_version", None)): + gateway_missing = missing_gateway_bindings(bindings) + if gateway_missing: + raise ProfileConfigurationError( + "INFERENCE_GATEWAY_UNBOUND", + "the release does not bind the research inference gateway: " + + ", ".join(gateway_missing), + "release_id", + ) + # A ``gateway`` tools binding sets nothing on the agent: only the + # research inference gateway can apply it, and only gateway-bound + # runtimes call it. The release's own identity must say so too: a + # mis-declared release (say a Codex command with Goose runtime bindings) + # would never call the gateway, and its selection would govern nothing. + if _is_gateway_tools_binding(bindings.get("tools")) and not ( + requires_inference_gateway(getattr(profile, "framework_version", None)) + and _uniquely_gateway_bound(release, document) + ): + raise ProfileConfigurationError( + "BYOA_CONFIG_UNENFORCEABLE", + "the release enforces tools at the research inference gateway, which " + f"{framework!r} does not call; its tool selection would govern nothing", + "release_id", + ) + for field, binding in bindings.items(): + if field != "tools": + continue + if str(binding.get("format") or "string").strip().lower() not in ( + "csv", + "json", + ): + raise ProfileConfigurationError( + "BYOA_CONFIG_FORMAT_INVALID", + "a tools binding must use format 'csv' or 'json' to render a list", + "release_id", + ) + + unknown_tools = _profile_tools(profile) - set(tools_for_framework(framework)) + if unknown_tools: + raise ProfileConfigurationError( + "TOOLS_NOT_SUPPORTED", + f"tools_json contains tools unsupported by {framework}: " + + ", ".join(sorted(unknown_tools)), + "tools_json", + ) + + option = str(getattr(profile, "approval_policy", None) or "").strip().lower() + if option not in ALL_APPROVAL_OPTIONS: + raise ProfileConfigurationError( + "APPROVAL_POLICY_UNSUPPORTED", + "approval_policy must be one of " + ", ".join(ALL_APPROVAL_OPTIONS), + "approval_policy", + ) + + _validate_harness_settings(profile) + + +def _validate_harness_settings(profile: Any) -> None: + """Refuse invalid built-in runtime command/harness settings (D-01). + + The profile API validates each field's shape first; this is the merged-state + check shared by create, update and the study freeze (which also sees rows + written around the API), including the verify-command-is-not-blocked rule. + """ + denylist = getattr(profile, "commands_denylist", None) + if denylist is not None: + try: + validate_commands_denylist(denylist) + except ValueError as error: + raise ProfileConfigurationError( + "COMMANDS_DENYLIST_INVALID", str(error), "commands_denylist" + ) from error + timeout = getattr(profile, "command_timeout_seconds", None) + if timeout is not None: + try: + validate_command_timeout_seconds(timeout) + except ValueError as error: + raise ProfileConfigurationError( + "COMMAND_TIMEOUT_INVALID", str(error), "command_timeout_seconds" + ) from error + options = getattr(profile, "harness_options", None) + if options is not None: + try: + validate_harness_options(options, commands_denylist=denylist) + except ValueError as error: + raise ProfileConfigurationError( + "HARNESS_OPTIONS_INVALID", str(error), "harness_options" + ) from error + + +def _document_mode(release: Any, document: Optional[Mapping[str, Any]]) -> str: + """The release's distribution mode, resolved exactly as validation does.""" + mode = _release_mode(release) + if not mode and document is not None: + mode = str( + document.get("distribution_mode") or DistributionMode.PACKAGED.value + ).strip().upper() + return mode + + +def _declared_identity_tokens( + release: Any, document: Optional[Mapping[str, Any]] +) -> set[str]: + """Lower-cased agent identity names a BYOA release declares. + + ``agent_package`` and ``agent_id`` are compared as-is; ``agent_command`` is + reduced to its executable name (no directory, no launcher suffix). + """ + tokens: set[str] = set() + for attribute in ("agent_package", "agent_id", "agent_command"): + value = getattr(release, attribute, None) + if value is None and document is not None: + value = document.get(attribute) + token = str(value or "").strip().lower() + if attribute == "agent_command": + token = token.replace("\\", "/").rsplit("/", 1)[-1] + for suffix in _COMMAND_SUFFIXES: + if token.endswith(suffix): + token = token[: -len(suffix)] + if token: + tokens.add(token) + return tokens + + +def release_profile_configurability( + release: Any, *, release_json: Optional[Mapping[str, Any]] = None +) -> dict[str, list[str]]: + """Which profiles may pin ``release`` and which profile fields govern it. + + Derived with the same framework↔mode table and binding parser as + :func:`validate_profile_configuration`, so a field is advertised only when + the release declares a valid binding for it (a ``tools`` binding whose + format cannot render a list is still refused by validation as + ``BYOA_CONFIG_FORMAT_INVALID``): + + * ``compatible_frameworks`` — the ``framework_version`` values that may pin + the release. ``PACKAGED`` → the managed runtime. ``BYOA_EXTERNAL`` → the + one BYOA framework its declared identity names; when the identity names + none or several, every BYOA framework (never a guess). + * ``configurable_fields`` — profile field names that actually reach the + runtime: every managed field for ``PACKAGED``; exactly the release's valid + ``byoa_config`` binding fields for BYOA. Field names only, never binding + keys, environment variable names or commands. + * ``required_bindings_missing`` — for BYOA, the always-set profile fields + the release does not bind (a profile pinned to it would be rejected as + ``BYOA_CONFIG_UNMAPPED``); empty otherwise. + """ + document = _release_document(release, release_json) + mode = _document_mode(release, document) + candidates = [ + framework + for framework, framework_mode in FRAMEWORK_DISTRIBUTION_MODES.items() + if framework_mode == mode + ] + if mode != DistributionMode.BYOA_EXTERNAL.value: + return { + "compatible_frameworks": candidates, + "configurable_fields": ( + list(PACKAGED_CONFIGURABLE_FIELDS) + if mode == DistributionMode.PACKAGED.value + else [] + ), + "required_bindings_missing": [], + } + + tokens = _declared_identity_tokens(release, document) + declared = [framework for framework in candidates if framework in tokens] + bindings = _byoa_bindings(release, document) + # Only a release that uniquely names a gateway-bound framework is held to + # the gateway bindings; an ambiguous identity never gains requirements. + gateway = len(declared) == 1 and requires_inference_gateway(declared[0]) + required_missing = [field for field in BYOA_ALWAYS_SET_FIELDS if field not in bindings] + if gateway: + required_missing.extend(missing_gateway_bindings(bindings)) + return { + "compatible_frameworks": declared if len(declared) == 1 else candidates, + # Profile fields only: the runtime (gateway) bindings are filled by the + # plugin and are never offered as configurable profile fields. + "configurable_fields": [ + field + for field in BYOA_CONFIG_FIELDS + if field in bindings + and not ( + field == "tools" + and _is_gateway_tools_binding(bindings[field]) + and not _uniquely_gateway_bound(release, document) + ) + ], + "required_bindings_missing": required_missing, + "inference_gateway": gateway, + } + + +def parse_command_args(value: Any) -> list[str]: + """Normalize a stored ``agent_command_args`` value into a string list. + + The column is JSON text, but callers may hand a list (a freshly built + payload) or ``None``. Anything unparseable degrades to an empty list rather + than raising, because it is only ever an argument vector. + """ + if value is None: + return [] + if isinstance(value, (list, tuple)): + return [str(item) for item in value] + if isinstance(value, str): + try: + parsed = json.loads(value) + except (TypeError, ValueError): + return [] + if isinstance(parsed, list): + return [str(item) for item in parsed] + return [] + + +def _packaged_digest( + release: AgentReleaseV1, platform: Optional[tuple[str, str]] +) -> Optional[str]: + registry = AgentRegistry() + if not registry.register_release(release).accepted: + return None + resolver = RegistryReleaseResolver(registry, platform=platform) + return resolver.artifact_digest(release) + + +def _agent_config( + profile: Any, + release: Optional[AgentReleaseV1], + connection: Any, + funding_owner_user_id: Any = None, +) -> Optional[ResolvedAgentConfig]: + """Freeze the profile's non-secret executable config, when available. + + A lightweight stand-in (no ``model``/``tools_json``) has no executable config + to freeze, so this returns ``None``; such a revision is refused at execution + time rather than read from a mutable profile row. + """ + if getattr(profile, "model", None) is None: + return None + profile_id = getattr(profile, "profile_id", None) or getattr( + profile, "distribution_id", None + ) + if profile_id is None: + return None + return ResolvedAgentConfig( + profile_id=profile_id, + name=str(getattr(profile, "name", "") or ""), + model=str(getattr(profile, "model", "") or ""), + framework_version=str(getattr(profile, "framework_version", "") or ""), + tools_json=str(getattr(profile, "tools_json", "") or "[]"), + approval_policy=str(getattr(profile, "approval_policy", "") or ""), + max_steps=int(getattr(profile, "max_steps", 0) or 0), + temperature=getattr(profile, "temperature", None), + max_context_tokens=getattr(profile, "max_context_tokens", None), + connection_id=getattr(profile, "connection_id", None), + connection_label=getattr(connection, "label", None), + funding_owner_user_id=funding_owner_user_id, + ) + + +def resolve_distribution_view( + profile: Any, + release: Optional[AgentReleaseV1], + *, + distribution_id: Any = None, + platform: Optional[tuple[str, str]] = None, + connection: Any = None, + funding_owner_user_id: Any = None, +) -> DistributionResolution: + """Build the validator's view of one distribution. + + ``profile`` is an ``AgentProfile`` row (or a schema-shaped stand-in) owning + the pin (``profile_id``/``release_id``) and its executable config; the + distribution mode, package/command identity and argument vector come from + ``release`` alone. ``release`` is the already rehydrated registry release + for ``release_id``, or ``None`` when it does not exist. + + When ``profile`` also exposes the executable config (a real profile row), + that non-secret config is attached as ``agent_config`` so publication can + freeze it into the immutable revision. ``funding_owner_user_id`` is the study + owner whose connection grant funds execution. + """ + if distribution_id is None: + distribution_id = getattr(profile, "profile_id", None) + release_id = getattr(profile, "release_id", None) + agent_config = _agent_config( + profile, release, connection, funding_owner_user_id + ) + + if release is not None and release.is_byoa: + # A BYOA distribution is identity-pinned (from the release) and can + # never be verified: there is no artifact to bind evidence to. + identity_present = release.byoa_identity is not None + return DistributionResolution( + found=True, + distribution_id=distribution_id, + distribution_mode=DistributionMode.BYOA_EXTERNAL.value, + release_id=release.release_id, + agent_id=release.agent_id, + version=release.version, + agent_package=release.agent_package, + agent_command=release.agent_command, + agent_command_args=list(release.agent_command_args), + artifact_digest=None, + verified=False, + release_status=( + ReleaseResolutionStatus.RESOLVED + if identity_present + else ReleaseResolutionStatus.UNQUALIFIED + ), + message=( + None + if identity_present + else "BYOA distribution declares neither a package nor a command" + ), + agent_config=agent_config, + ) + + mode = ( + _release_mode(release) + if release is not None + else DistributionMode.PACKAGED.value + ) or DistributionMode.PACKAGED.value + view = DistributionResolution( + found=True, + distribution_id=distribution_id, + distribution_mode=mode, + release_id=release_id, + agent_package=None, + agent_command=None, + artifact_digest=None, + verified=False, + agent_config=agent_config, + ) + if not release_id: + return view + if release is None: + view.message = "release is not registered" + return view + + view.agent_id = release.agent_id + view.version = release.version + if release.qualification_status in _WITHDRAWN: + view.release_status = ReleaseResolutionStatus.WITHDRAWN + view.message = f"release is {release.qualification_status.value}" + return view + if release.qualification_status != QualificationStatus.QUALIFIED: + view.release_status = ReleaseResolutionStatus.UNQUALIFIED + view.message = "release is not qualified" + return view + + view.verified = True + view.release_status = ReleaseResolutionStatus.RESOLVED + view.artifact_digest = _packaged_digest(release, platform) + return view + + +def distribution_supported_platforms( + release: Optional[AgentReleaseV1], +) -> list[dict[str, str]]: + """The ``(os, arch)`` pairs a release publishes, in a stable order.""" + if release is None: + return [] + artifacts = sorted(release.artifacts, key=lambda item: (item.os, item.arch)) + return [{"os": artifact.os, "arch": artifact.arch} for artifact in artifacts] diff --git a/src/research/study/agents/enums.py b/src/research/study/agents/enums.py new file mode 100644 index 0000000..69f6d05 --- /dev/null +++ b/src/research/study/agents/enums.py @@ -0,0 +1,134 @@ +"""Closed vocabularies for the agent registry and capability contract (Issue 04). + +Every member is part of a persisted contract (release JSON, snapshot JSON, +registry responses), so members are additive only: renaming or removing a +member is a breaking change to already-stored releases and snapshots. + +``Fidelity`` is re-exported from :mod:`research.compatibility.enums` so the +registry shares the exact Issue 01 fidelity vocabulary instead of inventing a +parallel one. +""" + +from __future__ import annotations + +from enum import Enum + +from research.compatibility.enums import Fidelity # noqa: F401 - intentional re-export + +__all__ = [ + "CapabilityCoverageState", + "DistributionMode", + "DistributionSourceType", + "Fidelity", + "INFERENCE_GATEWAY_FRAMEWORKS", + "MANAGED_RUNTIME_FRAMEWORK", + "METERED_FRAMEWORKS", + "QualificationStatus", + "RegistryReasonCode", + "SnapshotCapabilityState", +] + +#: The one agent runtime the managed protocol runs (packaged, ACP-relayed). +#: Goose and Codex are BYOA identities: the participant installs them and the +#: plugin launches them through the research proxy. Goose is pointed at the +#: research inference gateway (shared server-held key, per-participant +#: budgets); Codex signs in with ChatGPT and is neither relayed nor metered. +MANAGED_RUNTIME_FRAMEWORK = "code4me2-agent" + +#: BYOA runtimes whose model calls go through the research inference gateway +#: (``/api/research/inference/v1/chat/completions``). Their releases must bind +#: the gateway runtime fields (see ``BYOA_RUNTIME_FIELDS``). +INFERENCE_GATEWAY_FRAMEWORKS: frozenset[str] = frozenset({"goose"}) + +#: Runtimes whose model calls spend from the study's server-held provider key +#: and are metered against the participant's budget: Goose (through the +#: research inference gateway) and the managed runtime (through the ACP relay). +#: Codex signs in with ChatGPT and is neither relayed nor metered. +METERED_FRAMEWORKS: frozenset[str] = frozenset({"goose", MANAGED_RUNTIME_FRAMEWORK}) + + +class DistributionMode(str, Enum): + """How a release's agent reaches the participant host. + + ``PACKAGED`` (the default) is the digest-pinned contract: the agent is a + packaged archive the plugin installs from its verified cache or the release's + exact ``download_url``, and the release carries a verified ``sha256`` per + platform. ``BYOA_EXTERNAL`` means + "bring your own agent": the participant installs the agent (Goose, Codex, ...) + and the release pins a command/package identity instead of an artifact digest. + The two modes are persisted on the release and embedded in every bootstrap + manifest so the plugin never silently mixes them. + """ + + PACKAGED = "PACKAGED" + BYOA_EXTERNAL = "BYOA_EXTERNAL" + + +class QualificationStatus(str, Enum): + """Producer tests derive QUALIFIED; DISABLED is an irreversible withdrawal. + + Older status values remain readable for historical records. + """ + + UNQUALIFIED = "UNQUALIFIED" + DRAFT = "DRAFT" + QUALIFIED = "QUALIFIED" + CONDITIONALLY_QUALIFIED = "CONDITIONALLY_QUALIFIED" + DISABLED = "DISABLED" + RETIRED = "RETIRED" + BLOCKED = "BLOCKED" + + +class DistributionSourceType(str, Enum): + """Where a release's distribution metadata came from.""" + + EXTERNAL_REGISTRY = "EXTERNAL_REGISTRY" + RESEARCH_OVERLAY = "RESEARCH_OVERLAY" + BUNDLED = "BUNDLED" + + +class SnapshotCapabilityState(str, Enum): + """Capability state captured in a runtime snapshot. + + ``UNKNOWN``, ``DECLARED``, ``UNAVAILABLE``, ``PARTIAL`` and ``BROKEN`` are + deliberately distinct from ``OBSERVED``: declared intent and observed + behavior are separate facts, and an absent observation is never promoted to + a supported one. + """ + + UNKNOWN = "UNKNOWN" + DECLARED = "DECLARED" + OBSERVED = "OBSERVED" + UNAVAILABLE = "UNAVAILABLE" + PARTIAL = "PARTIAL" + BROKEN = "BROKEN" + + +class CapabilityCoverageState(str, Enum): + """Aggregate observability of one capability across declared vs observed.""" + + OBSERVED = "OBSERVED" + DECLARED_ONLY = "DECLARED_ONLY" + UNKNOWN = "UNKNOWN" + UNAVAILABLE = "UNAVAILABLE" + PARTIAL = "PARTIAL" + BROKEN = "BROKEN" + + +class RegistryReasonCode(str, Enum): + """Stable machine-readable reason a registry operation was rejected.""" + + DIGEST_MISMATCH = "DIGEST_MISMATCH" + UNSUPPORTED_PLATFORM = "UNSUPPORTED_PLATFORM" + NOT_QUALIFIED = "NOT_QUALIFIED" + WITHDRAWN = "WITHDRAWN" + DUPLICATE_RELEASE = "DUPLICATE_RELEASE" + INVALID_TRANSITION = "INVALID_TRANSITION" + RELEASE_NOT_FOUND = "RELEASE_NOT_FOUND" + ARTIFACT_MISSING = "ARTIFACT_MISSING" + INCOMPLETE_ARTIFACTS = "INCOMPLETE_ARTIFACTS" + ADAPTER_INCOMPATIBLE = "ADAPTER_INCOMPATIBLE" + INVALID_VERSION_RANGE = "INVALID_VERSION_RANGE" + SNAPSHOT_INVALID = "SNAPSHOT_INVALID" + # A BYOA release declares neither a command nor a package identity to resolve. + AGENT_NOT_FOUND = "AGENT_NOT_FOUND" diff --git a/src/research/study/agents/manifest_import.py b/src/research/study/agents/manifest_import.py new file mode 100644 index 0000000..8242d7b --- /dev/null +++ b/src/research/study/agents/manifest_import.py @@ -0,0 +1,473 @@ +"""Import a *build* runtime manifest plus its archive bytes into a release. + +The packaging pipeline emits a JSON build manifest (``manifest_version``, +``runtime_version``, ``server_commit``/``plugin_commit`` and ``artifacts[]``) +whose ``archive`` fields are **basenames**. This module turns that document into +the registry's own release contract so a fresh database can be made immediately +usable without a human hand-typing a digest: + +* the release identity (``agent_id``/``release_id``/``version``) is derived + deterministically from the manifest, so re-importing identical bytes resolves + the *same* release instead of piling up duplicates; +* ``source_manifest_digest`` is the SHA-256 of the canonical manifest bytes and + identifies the **release record**; +* each built ``artifacts[]`` entry is bound to the **exact uploaded archive** + whose digest and size are recomputed here -- a mismatch or a placeholder + (all-zero) digest rejects the *entire* import and no release row is created. + +Only bytes are accepted: there is no artifact-root lookup and no caller-supplied +size. A manifest that declares an archive the caller did not upload (or an upload +the manifest does not declare) is rejected; a platform the build did not produce +is *omitted from the manifest entirely*, never represented by a placeholder. + +The module is deliberately free of database/App dependencies: the API router and +any local-dev seeding share exactly this planning step. +""" + +from __future__ import annotations + +import hashlib +import json +from urllib.parse import urlsplit, unquote +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, Optional + +from pydantic import ValidationError + +from .enums import DistributionMode, DistributionSourceType, QualificationStatus +from .models import ( + AdapterRef, + AgentReleaseV1, + DistributionArtifact, + ReleaseDisplay, + ReleaseTests, +) + +__all__ = [ + "PLACEHOLDER_DIGEST", + "ManifestImportError", + "ManifestImportPlan", + "build_manifest_release", + "canonical_manifest_bytes", + "manifest_digest", + "sha256_prefixed", +] + +#: Sentinel digest a build manifest must never carry for a shipped archive. +PLACEHOLDER_DIGEST = "0" * 64 + +#: The only GitHub repository (owner, name; GitHub compares them case-insensitively) +#: whose release assets participants' plugins may download. The plugin enforces +#: the same pin in ManagedRuntimeInstaller. +RUNTIME_RELEASE_REPOSITORY = ("aise-tudelft", "code4me2-server") + + +class ManifestImportError(ValueError): + """A typed, operator-facing reason a build manifest cannot be imported. + + ``code``/``field`` mirror the registry's typed issue shape so the endpoint + can return a machine-readable rejection. + """ + + def __init__(self, code: str, message: str, field: str = "") -> None: + super().__init__(message) + self.code = code + self.message = message + self.field = field + + +@dataclass(frozen=True) +class ManifestImportPlan: + """The release a build manifest + its archives resolve to.""" + + release: AgentReleaseV1 + manifest_digest: str + verified_artifacts: list[dict[str, Any]] = field(default_factory=list) + byoa_releases: list[AgentReleaseV1] = field(default_factory=list) + + +def canonical_manifest_bytes(manifest: Mapping[str, Any]) -> bytes: + """Canonical UTF-8 bytes of a manifest mapping. + + Keys are sorted and separators are compact so the digest is stable for the + same semantic document regardless of formatting/whitespace: a re-import of + the "same" manifest is idempotent and never a new release. + """ + return json.dumps( + manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("utf-8") + + +def manifest_digest(manifest: Mapping[str, Any]) -> str: + """``sha256:`` of the canonical manifest bytes (release identity).""" + return sha256_prefixed(canonical_manifest_bytes(manifest)) + + +def sha256_prefixed(data: bytes) -> str: + """``sha256:`` for ``data`` (the repo-wide digest convention).""" + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def _bare_digest(value: Any) -> Optional[str]: + """Return the bare lowercase hex of a digest, or ``None`` if malformed.""" + text = str(value or "").strip().lower() + if text.startswith("sha256:"): + text = text[len("sha256:") :] + if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text): + return None + return text + + +def _normalize_os(value: Any) -> str: + """Map a manifest platform name onto the repo's ``macos|linux|windows``.""" + text = str(value or "").strip().lower() + if text in {"macos", "darwin", "mac", "macosx"}: + return "macos" + if text.startswith("win"): + return "windows" + if text.startswith("linux"): + return "linux" + raise ManifestImportError( + "UNSUPPORTED_PLATFORM", f"unknown platform {value!r}", "platform" + ) + + +def _normalize_arch(value: Any) -> str: + """Map a manifest architecture onto the repo's ``arm64|x64`` vocabulary.""" + text = str(value or "").strip().lower() + if text in {"arm64", "aarch64"}: + return "arm64" + if text in {"x64", "x86_64", "amd64"}: + return "x64" + raise ManifestImportError( + "UNSUPPORTED_PLATFORM", f"unknown architecture {value!r}", "architecture" + ) + + +def _as_size(value: Any) -> Optional[int]: + """Coerce a size-like value to a positive int, or ``None``.""" + if isinstance(value, bool) or value is None: + return None + try: + size = int(value) + except (TypeError, ValueError): + return None + return size if size > 0 else None + + +def _file_digest_and_size(path: Path) -> tuple[str, int]: + digest = hashlib.sha256() + size = 0 + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + size += len(chunk) + digest.update(chunk) + return digest.hexdigest(), size + + +def _archive_basename(value: Any, field: str) -> str: + """A declared archive must be a bare basename (no directory component).""" + text = str(value or "").strip() + if not text: + raise ManifestImportError( + "INVALID_MANIFEST", "artifact declares no archive name", field + ) + if text in {".", ".."} or text != Path(text).name or "/" in text or "\\" in text: + raise ManifestImportError( + "INVALID_MANIFEST", + ( + f"artifact archive {text!r} must be a bare archive basename; " + "the producer manifest never carries a resource directory prefix" + ), + field, + ) + return text + + +def _derive_identity( + manifest: Mapping[str, Any], digest: str +) -> tuple[str, str, str]: + """Deterministically derive ``(agent_id, release_id, version)``.""" + artifacts = [item for item in manifest.get("artifacts") or [] if isinstance(item, Mapping)] + runtime_ids = sorted( + {str(item.get("runtime_id")).strip() for item in artifacts if item.get("runtime_id")} + ) + if len(runtime_ids) > 1: + raise ManifestImportError( + "INVALID_MANIFEST", + "manifest artifacts disagree on runtime_id: " + + ", ".join(runtime_ids), + "artifacts.runtime_id", + ) + agent_id = runtime_ids[0] if runtime_ids else "code4me-agent" + + version = str( + manifest.get("runtime_version") or manifest.get("version") or "" + ).strip() + if not version: + raise ManifestImportError( + "INVALID_MANIFEST", + "manifest declares no runtime_version", + "runtime_version", + ) + + # The digest is folded into the release id so a changed manifest is a + # *distinct* release, exactly as the registry's digest identity requires. + digest_hex = digest[len("sha256:") :] + release_id = f"{agent_id}-{version}-{digest_hex[:12]}" + return agent_id, release_id, version + + +def _derive_adapter( + manifest: Mapping[str, Any], *, agent_id: str, version: str +) -> Optional[AdapterRef]: + """The adapter identity the manifest declares, or ``None``. + + A manifest that carries an explicit ``adapter``/``adapter_ref`` block keeps + it as-is (including its own ``digest``). Otherwise the release declares **no + adapter**: the server never invents an adapter identity or reuses the + manifest/release digest as one, and nothing gates on the adapter anyway. + """ + explicit = manifest.get("adapter") or manifest.get("adapter_ref") + if isinstance(explicit, Mapping): + return AdapterRef.model_validate(dict(explicit)) + return None + + +def build_manifest_release( + manifest: Mapping[str, Any], + *, + archives: Optional[Mapping[str, Path]] = None, +) -> ManifestImportPlan: + """Build the :class:`AgentReleaseV1` a build manifest + archives describe. + + ``archives`` maps each declared archive **basename** to the on-disk file the + caller uploaded. Every declared archive must be present exactly once, every + uploaded file must be declared, and each file's size and SHA-256 are + recomputed and must match the manifest. Any failure rejects the whole import. + """ + if not isinstance(manifest, Mapping): + raise ManifestImportError( + "INVALID_MANIFEST", "manifest must be a JSON object", "manifest" + ) + if manifest.get("manifest_version") not in (None, 1, "1"): + raise ManifestImportError( + "INVALID_MANIFEST", "unsupported manifest_version", "manifest_version" + ) + raw_artifacts = manifest.get("artifacts") + if not isinstance(raw_artifacts, list) or not raw_artifacts: + raise ManifestImportError( + "INVALID_MANIFEST", + "manifest declares no artifacts", + "artifacts", + ) + + digest = manifest_digest(manifest) + agent_id, release_id, version = _derive_identity(manifest, digest) + supplied = dict(archives or {}) + + artifacts: list[DistributionArtifact] = [] + tests: list[ReleaseTests] = [] + verified: list[dict[str, Any]] = [] + declared_names: set[str] = set() + seen_platforms: set[tuple[str, str]] = set() + + for index, raw in enumerate(raw_artifacts): + if not isinstance(raw, Mapping): + raise ManifestImportError( + "INVALID_MANIFEST", + "each artifacts[] entry must be an object", + f"artifacts[{index}]", + ) + raw_platform = raw.get("platform") + raw_arch = raw.get("architecture") + os_name = _normalize_os(raw_platform) + arch = _normalize_arch(raw_arch) + platform = f"{os_name}-{arch}" + if (os_name, arch) in seen_platforms: + raise ManifestImportError( + "INVALID_MANIFEST", + f"duplicate artifact for platform {platform!r}", + f"artifacts[{index}]", + ) + seen_platforms.add((os_name, arch)) + try: + tests.append(ReleaseTests.model_validate(dict(raw.get("tests") or {}, os=os_name, arch=arch))) + except (ValidationError, TypeError, ValueError) as error: + raise ManifestImportError( + "TESTS_NOT_PASSED", "each platform requires passing self_check and acp_initialize with ran_at", + f"artifacts[{index}].tests", + ) from error + + + archive = _archive_basename(raw.get("archive"), f"artifacts[{index}].archive") + if archive in declared_names: + raise ManifestImportError( + "DUPLICATE_ARCHIVE", + f"manifest declares archive {archive!r} more than once", + f"artifacts[{index}].archive", + ) + declared_names.add(archive) + + declared = _bare_digest(raw.get("sha256")) + if declared is None or declared == PLACEHOLDER_DIGEST: + raise ManifestImportError( + "PLACEHOLDER_DIGEST", + ( + f"artifact {archive!r} declares a placeholder or malformed " + "sha256; a shipped archive must carry its real digest" + ), + f"artifacts[{index}].sha256", + ) + declared_size = _as_size(raw.get("size")) + if declared_size is None: + raise ManifestImportError( + "INVALID_MANIFEST", + f"artifact {archive!r} declares no positive size", + f"artifacts[{index}].size", + ) + download_url = raw.get("download_url") + if download_url is not None: + try: + url = urlsplit(str(download_url)) + except ValueError: + raise ManifestImportError("DOWNLOAD_URL_INVALID", "Agent downloads require an exact public GitHub release asset URL of AISE-TUDelft/code4me2-server.", "download_url") from None + segments = [unquote(part) for part in url.path.split("/")[1:]] + if not ( + url.scheme == "https" and url.netloc == "github.com" and not url.query and not url.fragment + and len(segments) == 6 and all(part and part not in {".", ".."} and "/" not in part for part in segments) + and tuple(part.lower() for part in segments[:2]) == RUNTIME_RELEASE_REPOSITORY + and segments[2:4] == ["releases", "download"] and segments[4] != "latest" and segments[-1] == archive + ): + raise ManifestImportError("DOWNLOAD_URL_INVALID", "Agent downloads require an exact public GitHub release asset URL of AISE-TUDelft/code4me2-server.", "download_url") + artifacts.append( + DistributionArtifact( + os=os_name, + arch=arch, + path=archive, + sha256="sha256:" + declared, + size=declared_size, + download_url=download_url, + managed_protocol=raw.get("managed_protocol") or manifest.get("managed_protocol_version"), + executable=( + str(raw.get("executable")).strip() + if raw.get("executable") + else None + ), + ) + ) + + unexpected = sorted(set(supplied) - declared_names) + if unexpected: + raise ManifestImportError( + "UNEXPECTED_ARCHIVE", + "uploaded archive(s) are not declared by the manifest: " + + ", ".join(unexpected), + "archives", + ) + missing = sorted(declared_names - set(supplied)) + if missing: + raise ManifestImportError( + "ARTIFACT_MISSING", + "declared archive(s) were not uploaded: " + ", ".join(missing), + "archives", + ) + + for artifact in artifacts: + source = supplied[artifact.path] + if not Path(source).is_file(): + raise ManifestImportError( + "ARTIFACT_MISSING", + f"uploaded archive {artifact.path!r} is not readable", + "archives", + ) + actual, size = _file_digest_and_size(Path(source)) + expected = _bare_digest(artifact.sha256) + if actual != expected: + raise ManifestImportError( + "DIGEST_MISMATCH", + ( + f"archive {artifact.path!r} digest {actual} does not match the " + f"manifest's {expected}" + ), + "archives", + ) + if size != artifact.size: + raise ManifestImportError( + "SIZE_MISMATCH", + ( + f"archive {artifact.path!r} size {size} does not match the " + f"manifest's {artifact.size}" + ), + "archives", + ) + verified.append( + { + "archive": artifact.path, + "platform": f"{artifact.os}-{artifact.arch}", + "sha256": "sha256:" + actual, + "size": size, + "verified": True, + } + ) + + release = AgentReleaseV1( + agent_id=agent_id, + release_id=release_id, + version=version, + display=ReleaseDisplay( + name=f"Code4Me agent runtime {version}", + vendor="code4me2", + description="Imported from the built runtime manifest and verified archives.", + ), + source_type=DistributionSourceType.BUNDLED, + source_manifest_digest=digest, + artifacts=artifacts, + min_plugin_version=manifest.get("min_plugin_version"), + min_protocol_version=manifest.get("min_protocol_version"), + max_protocol_version=manifest.get("max_protocol_version"), + tests=tests, + qualification_status=QualificationStatus.QUALIFIED, + adapter=_derive_adapter(manifest, agent_id=agent_id, version=version), + created_at=datetime.now(timezone.utc), + ) + byoa_releases = [] + seen_agents = {agent_id} + raw_agents = manifest.get("agents", []) + if not isinstance(raw_agents, list): + raise ManifestImportError("INVALID_MANIFEST", "agents must be an array", "agents") + for index, raw in enumerate(raw_agents): + try: + from .participant_release import AgentInput + + agent = AgentInput.model_validate(raw) + if agent.framework == "code4me2-agent" or agent.framework in seen_agents: + raise ValueError("duplicate or managed agent in external declarations") + seen_agents.add(agent.framework) + if not agent.tests: + raise ValueError("external agents need their own passing platform tests") + byoa_releases.append(AgentReleaseV1( + agent_id=agent.framework, + release_id=f"{agent.framework}-{agent.version}-{digest.removeprefix('sha256:')[:12]}", + version=agent.version, + source_manifest_digest=digest, + distribution_mode=DistributionMode.BYOA_EXTERNAL, + agent_command=agent.agent_command, + agent_command_args=agent.agent_command_args, + agent_package=agent.framework, + adapter=agent.adapter, + byoa_config=agent.byoa_config, + tests=agent.tests, + qualification_status=QualificationStatus.QUALIFIED, + created_at=release.created_at, + )) + except (ValidationError, TypeError, ValueError) as error: + raise ManifestImportError("INVALID_MANIFEST", str(error), f"agents[{index}]") from error + return ManifestImportPlan( + byoa_releases=byoa_releases, + release=release, + manifest_digest=digest, + verified_artifacts=verified, + ) diff --git a/src/research/study/agents/models.py b/src/research/study/agents/models.py new file mode 100644 index 0000000..c7d83ab --- /dev/null +++ b/src/research/study/agents/models.py @@ -0,0 +1,442 @@ +"""Pydantic v2 contracts for the agent registry and capability contract. + +Two documents are defined: + +* :class:`AgentReleaseV1` - the immutable distribution/release identity a study + condition pins. Its digest identity is + ``(agent_id, release_id, source_manifest_digest)``: the same agent version + with a different source manifest digest is a *distinct* release, never an + overwrite. +* :class:`CapabilitySnapshotV1` - per-run declared-vs-observed capability + evidence. ``declared`` and ``observed`` are kept in separate maps so a + declared capability that was never observed stays ``DECLARED`` with an + explicit ``UNAVAILABLE``/``UNKNOWN`` observation. A missing measurement is + ``value is None`` plus a coverage state, never ``0``, ``False`` or success. +""" + +from __future__ import annotations + +import re +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Any, Literal, Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from research.compatibility.enums import Fidelity + +from .enums import ( + CapabilityCoverageState, + DistributionMode, + DistributionSourceType, + QualificationStatus, + SnapshotCapabilityState, +) + +#: Frozen profile fields a BYOA release may declare a translation for. A field +#: the profile actually sets must be covered by a binding (ISSUE-03 Path A). +BYOA_CONFIG_FIELDS = ( + "model", + "temperature", + "max_steps", + "tools", + "approval_policy", +) +#: Runtime fields the *plugin* fills for a BYOA agent that calls the research +#: inference gateway (Goose): the gateway origin and path, the per-participant +#: credential, the provider kind the agent must select and an isolated state +#: directory. They are never profile fields (an admin never sees them as +#: configurable), and the credential binding must use the ``env`` transport: +#: argv is world-readable. +BYOA_RUNTIME_FIELDS = ( + "inference_gateway_host", + "inference_gateway_base_path", + "inference_gateway_credential", + "provider_kind", + "state_dir", +) +BYOA_CREDENTIAL_FIELD = "inference_gateway_credential" +BYOA_PROVIDER_KIND_FIELD = "provider_kind" +#: The provider kind value the server emits; the release maps it to the agent's +#: own vocabulary through ``value_map`` (Goose: ``openai``). +BYOA_PROVIDER_KIND_OPENAI_COMPATIBLE = "openai_compatible" +BYOA_BINDING_FIELDS = BYOA_CONFIG_FIELDS + BYOA_RUNTIME_FIELDS +#: ``gateway`` is not a launch transport: it declares that the research inference +#: gateway enforces the field (only ``tools``, only for gateway-bound runtimes). +BYOA_CONFIG_TRANSPORTS = ("env", "arg", "gateway") +BYOA_GATEWAY_TOOLS_KEY = "tool_allowlist" +BYOA_CONFIG_FORMATS = ("string", "json", "csv") + +_BASE_CONFIG = ConfigDict(extra="forbid") +_FROZEN_CONFIG = ConfigDict(extra="forbid", frozen=True) + +_AARCH64_ARCHS = frozenset({"aarch64", "arm64"}) +_X64_ARCHS = frozenset({"x86_64", "amd64", "x64"}) + + +def normalize_platform(os_name: str, arch: str) -> tuple[str, str]: + """Canonicalise an ``(os, arch)`` pair for platform comparison. + + Hosts report the same platform under different names (``Darwin`` vs + ``macos``, ``AMD64`` vs ``x64``), so the requested tuple and every stored + artifact tuple are normalised before comparison instead of blocking on a + cosmetic string mismatch. + """ + lowered_os = (os_name or "").strip().lower() + if "mac" in lowered_os or lowered_os == "darwin": + canonical_os = "macos" + elif lowered_os.startswith("win"): + canonical_os = "windows" + elif "linux" in lowered_os: + canonical_os = "linux" + else: + canonical_os = lowered_os + + lowered_arch = (arch or "").strip().lower() + if lowered_arch in _AARCH64_ARCHS: + canonical_arch = "arm64" + elif lowered_arch in _X64_ARCHS: + canonical_arch = "x64" + else: + canonical_arch = lowered_arch + + return (canonical_os, canonical_arch) + + +class ReleaseDisplay(BaseModel): + """Human-facing, non-secret release metadata.""" + + model_config = _BASE_CONFIG + + name: Optional[str] = None + vendor: Optional[str] = None + description: Optional[str] = None + homepage: Optional[str] = None + + +class ReleaseTests(BaseModel): + """Producer results for one native platform, bound by the manifest digest.""" + + model_config = _BASE_CONFIG + + os: str + arch: str + self_check: Literal["PASS"] + acp_initialize: Literal["PASS"] + ran_at: datetime + + @model_validator(mode="after") + def canonical_platform(self): + self.os, self.arch = normalize_platform(self.os, self.arch) + if self.os not in {"macos", "linux", "windows"} or self.arch not in {"arm64", "x64"}: + raise ValueError("unsupported test platform") + if self.ran_at.tzinfo is None: + raise ValueError("ran_at must include a timezone") + return self + + +class DistributionArtifact(BaseModel): + """One platform-specific distribution artifact of a release. + + ``path`` is deliberately *relative*: releases must never carry a local + absolute path or host-specific mutable registry configuration. The artifact's + identity is its archive digest (the plugin's bootstrap pin) plus, for the + managed runtime, the adapter name and the executable name. + """ + + model_config = _BASE_CONFIG + + os: str + arch: str + path: str + sha256: str + size: int + executable: Optional[str] = None + download_url: Optional[str] = None + managed_protocol: Optional[str] = None + license: Optional[str] = None + license_review: Optional[str] = None + signature: Optional[str] = None + + @property + def platform(self) -> tuple[str, str]: + """The ``(os, arch)`` platform this artifact targets.""" + return (self.os, self.arch) + + +class AdapterRef(BaseModel): + """Identity of the adapter/packaging layer that drives one release.""" + + model_config = _BASE_CONFIG + + adapter_id: str + version: str + digest: Optional[str] = None + # Declared ranges (e.g. ``">=1.2.0,<2.0.0"``) the adapter claims to drive. + supported_release_ranges: list[str] = Field(default_factory=list) + + +class AgentConfigBinding(BaseModel): + """One declared translation of a frozen profile field for a BYOA release. + + The release owns the external agent's configuration vocabulary, so it + declares how each frozen profile field reaches the process at launch: + + * ``transport="env"`` sets the environment variable ``key``; + * ``transport="arg"`` appends ``[key, value]`` to the agent argv; + * ``transport="gateway"`` (``tools`` only, key ``tool_allowlist``, JSON) + sets nothing on the agent: the research inference gateway offers the + model only the frozen selection. For an agent such as Goose, which reads + no tool setting from its environment, this is the only enforcement point, + so profile validation accepts it only for gateway-bound runtimes. + + ``format`` renders list values (``tools``) as ``csv`` or ``json``; + ``value_map`` translates the server-side vocabulary to the agent's (for + example ``per_step -> on-request``). A binding whose transport is not + executable fails closed at profile/study creation; it is never silently + ignored. + """ + + model_config = _BASE_CONFIG + + field: str + transport: Literal["env", "arg", "gateway"] + key: str + format: Literal["string", "json", "csv"] = "string" + value_map: dict[str, str] = Field(default_factory=dict) + + @field_validator("field") + @classmethod + def _known_field(cls, value: str) -> str: + normalized = (value or "").strip().lower() + if normalized not in BYOA_BINDING_FIELDS: + raise ValueError("field must be one of " + ", ".join(BYOA_BINDING_FIELDS)) + return normalized + + @field_validator("key") + @classmethod + def _non_blank_key(cls, value: str) -> str: + normalized = (value or "").strip() + if not normalized: + raise ValueError("key must not be blank") + if "=" in normalized or any( + ord(char) < 32 or ord(char) == 127 for char in normalized + ): + # ``--agent-env`` is a KEY=VALUE token: a key containing '=' or a + # control character would silently mis-bind or corrupt the launch + # argv. + raise ValueError("key must not contain '=' or control characters") + return normalized + + @model_validator(mode="after") + def _transport_key_shape(self) -> AgentConfigBinding: + if self.transport == "gateway" and ( + self.field != "tools" + or self.key != BYOA_GATEWAY_TOOLS_KEY + or self.format != "json" + ): + raise ValueError( + "a gateway binding is only valid for tools, with key " + f"'{BYOA_GATEWAY_TOOLS_KEY}' and format 'json'" + ) + if self.transport == "env" and not re.fullmatch( + r"[A-Za-z_][A-Za-z0-9_]*", self.key + ): + raise ValueError( + "an env binding key must be a valid environment variable name" + ) + if self.field == BYOA_CREDENTIAL_FIELD and self.transport != "env": + raise ValueError( + "the inference_gateway_credential binding must use the env " + "transport: a command-line argument is world-readable" + ) + return self + + +class AgentReleaseV1(BaseModel): + """An immutable, digest-pinned agent release record (schema version 1). + + ``distribution_mode`` selects the distribution contract: ``PACKAGED`` + (default) requires at least one digest-pinned artifact, while + ``BYOA_EXTERNAL`` (participant-installed) requires a command/package identity + and carries no artifact digest. The mode is additive so already-stored + releases without the field continue to parse as ``PACKAGED``. + """ + + model_config = _BASE_CONFIG + + schema_version: str = "1" + agent_id: str + # Stable, unique release identifier. It is part of the digest identity and + # must be changed when the upstream manifest digest changes. + release_id: str + version: str + display: ReleaseDisplay = Field(default_factory=ReleaseDisplay) + source_type: DistributionSourceType = DistributionSourceType.RESEARCH_OVERLAY + source_manifest_digest: str + distribution_mode: DistributionMode = DistributionMode.PACKAGED + artifacts: list[DistributionArtifact] = Field(default_factory=list) + # BYOA only: the explicit participant-installed agent command (an executable + # name or a portable relative reference; never a local absolute path). A + # participant host may override it with a configured path. + agent_command: Optional[str] = None + agent_command_args: list[str] = Field(default_factory=list) + # BYOA only: the logical package (``goose``/``codex``/...) used for discovery + # when no explicit command is pinned. + agent_package: Optional[str] = None + # BYOA only: the declared translation of frozen profile fields into the + # external agent's configuration (ISSUE-03 Path A). Empty means the release + # cannot honor any profile field, so a profile that sets one is rejected. + byoa_config: list[AgentConfigBinding] = Field(default_factory=list) + adapter: Optional[AdapterRef] = None + # Negotiated ACP protocol range the release is compatible with. + min_protocol_version: Optional[str] = None + max_protocol_version: Optional[str] = None + min_plugin_version: Optional[str] = None + tests: list[ReleaseTests] = Field(default_factory=list) + qualification_status: QualificationStatus = QualificationStatus.UNQUALIFIED + created_at: Optional[datetime] = None + + @model_validator(mode="after") + def _unique_config_bindings(self) -> AgentReleaseV1: + seen: set[str] = set() + env_keys: set[str] = set() + for binding in self.byoa_config: + if binding.field in seen: + raise ValueError( + f"duplicate byoa_config binding for {binding.field!r}" + ) + seen.add(binding.field) + if binding.transport == "env": + if binding.key in env_keys: + raise ValueError( + f"two byoa_config bindings set the same environment variable {binding.key!r}" + ) + env_keys.add(binding.key) + if self.byoa_config and not self.is_byoa: + raise ValueError("byoa_config is only valid for a BYOA_EXTERNAL release") + return self + + @property + def digest_identity(self) -> tuple[str, str, str]: + """The identity that makes two release records the same release.""" + return (self.agent_id, self.release_id, self.source_manifest_digest) + + @property + def is_byoa(self) -> bool: + """True when the agent is participant-installed rather than packaged.""" + return self.distribution_mode == DistributionMode.BYOA_EXTERNAL + + @property + def byoa_identity(self) -> Optional[str]: + """The BYOA command/package identity, or ``None`` for a packaged release.""" + if not self.is_byoa: + return None + return ( + (self.agent_command or "").strip() + or (self.agent_package or "").strip() + or None + ) + + def artifact_for(self, os_name: str, arch: str) -> Optional[DistributionArtifact]: + """Return the exact artifact for ``(os_name, arch)`` or ``None``. + + There is intentionally no fallback: an unavailable platform must block + before download/launch rather than silently selecting another artifact. + """ + requested = normalize_platform(os_name, arch) + for artifact in self.artifacts: + if normalize_platform(artifact.os, artifact.arch) == requested: + return artifact + return None + + +class CapabilityEntry(BaseModel): + """One capability measurement or declaration. + + ``value`` is nullable and is *not* defaulted: a missing measurement is + ``None`` and its meaning is carried by ``state``, the fidelity and the + limitations. It is never coerced to ``0``, ``False`` or success. + """ + + model_config = _FROZEN_CONFIG + + state: SnapshotCapabilityState + value: Optional[Any] = None + fidelity: Fidelity = Fidelity.NORMALIZED + source: str = "handshake" + limitations: list[str] = Field(default_factory=list) + observed_at: Optional[datetime] = None + + +class EnvironmentRef(BaseModel): + """Opaque reference to the environment a snapshot was captured against.""" + + model_config = _BASE_CONFIG + + environment_id: Optional[str] = None + host_kind: Optional[str] = None + os: Optional[str] = None + arch: Optional[str] = None + ide_build: Optional[str] = None + plugin_version: Optional[str] = None + + +class SnapshotFailure(BaseModel): + """Failure captured while producing a snapshot, if any.""" + + model_config = _BASE_CONFIG + + reason: str + detail: Optional[str] = None + occurred_at: Optional[datetime] = None + + +class CapabilitySnapshotV1(BaseModel): + """Immutable declared-vs-observed capability snapshot (schema version 1).""" + + model_config = _FROZEN_CONFIG + + schema_version: str = "1" + snapshot_id: UUID + release_id: str + agent_id: str + adapter_id: Optional[str] = None + adapter_version: Optional[str] = None + environment: EnvironmentRef = Field(default_factory=EnvironmentRef) + protocol_version: str + # Declared and observed are separate maps keyed by capability name. Unknown + # capability names are preserved rather than dropped. + declared: dict[str, CapabilityEntry] = Field(default_factory=dict) + observed: dict[str, CapabilityEntry] = Field(default_factory=dict) + evidence_refs: list[str] = Field(default_factory=list) + captured_at: datetime + failure: Optional[SnapshotFailure] = None + + +class CapabilityCoverage(BaseModel): + """Declared-vs-observed coverage for one capability. + + Both sides keep their explicit state (including ``UNKNOWN`` and + ``UNAVAILABLE``); the summary never collapses them into a boolean. + """ + + model_config = _BASE_CONFIG + + capability: str + declared_state: SnapshotCapabilityState = SnapshotCapabilityState.UNKNOWN + observed_state: SnapshotCapabilityState = SnapshotCapabilityState.UNKNOWN + coverage: CapabilityCoverageState + value_present: bool = False + limitations: list[str] = Field(default_factory=list) + + +class CapabilityCoverageReport(BaseModel): + """Coverage summary for one snapshot, with explicit state counts.""" + + model_config = _BASE_CONFIG + + snapshot_id: UUID + release_id: str + entries: list[CapabilityCoverage] = Field(default_factory=list) + counts: dict[str, int] = Field(default_factory=dict) diff --git a/src/research/study/agents/participant_release.py b/src/research/study/agents/participant_release.py new file mode 100644 index 0000000..d1282bd --- /dev/null +++ b/src/research/study/agents/participant_release.py @@ -0,0 +1,454 @@ +"""The one participant-release recipe: build inputs in, a single recipe out. + +No database, no downloads and no agent installation. The producer (the +``scripts/participant-release.py`` CLI, run locally or by CI) builds the native +runtime ZIPs, runs the agent self-check, and calls :func:`prepare` to fold +everything into **one recipe document**: + +* the runtime build manifest plus the exact ZIP ``sha256``/``size`` per platform; +* the adapter identity; +* the BYOA agent declarations (Goose, Codex, ...); +* the self-check verdict (``tests``). + +The recipe is the single source of truth: the server imports it, verifying its +declared ZIPs against the archives. The plugin receives a package-path +projection of the recipe whose archive names point at its bundled resources. +There is no extracted-file inventory. A self-check that did not pass produces +no recipe. +""" +from __future__ import annotations + +import hashlib +import json +import re +import shutil +from pathlib import Path +from typing import Any, Literal, Optional, Sequence +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from .models import AdapterRef, AgentConfigBinding, ReleaseTests, normalize_platform + +#: Canonical native platform ids. ``arm64`` is the single canonical spelling for +#: the 64-bit ARM architecture (``aarch64`` is normalised to it at the boundary). +PLATFORMS = ("macos-arm64", "macos-x64", "linux-x64", "windows-x64") +SEMVER = r"^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$" + +#: The one architecture spelling accepted in a recipe. +_CANONICAL_ARCH = {"arm64", "x64"} + +#: Every accepted architecture spelling, normalised to the single canonical one. +_ARCH_ALIASES = { + "arm64": "arm64", + "aarch64": "arm64", + "x64": "x64", + "x86_64": "x64", + "amd64": "x64", +} + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def canonical_platform(value: str) -> str: + """Normalise ``os-arch`` to the single canonical spelling (``arm64``/``x64``). + + ``aarch64``/``arm64`` (and ``x86_64``/``amd64``/``x64``) are all accepted at + the boundary and folded to the one canonical spelling, so a recipe never + carries two names for the same platform. + """ + if "-" not in value: + raise ValueError(f"platform {value!r} must be 'os-arch'") + os_name, arch = value.split("-", 1) + canonical_os, _ = normalize_platform(os_name, arch) + canonical_arch = _ARCH_ALIASES.get(arch.strip().lower()) + if canonical_os not in {"macos", "linux", "windows"} or canonical_arch not in _CANONICAL_ARCH: + raise ValueError(f"platform {value!r} is not a supported native platform") + return f"{canonical_os}-{canonical_arch}" + + +def input_path(root: Path, value: str) -> Path: + # Recipes are portable: machine paths are command options, never recipe data. + result = (root / value).resolve() + if not result.is_relative_to(root.resolve()): + raise ValueError("input escapes the recipe input directory") + return result + + +class AgentInput(BaseModel): + model_config = ConfigDict(extra="forbid") + framework: Literal["code4me2-agent", "goose", "codex"] + version: str = Field(min_length=1) + adapter: Optional[AdapterRef] = None + agent_command: str | None = None + agent_command_args: list[str] = Field(default_factory=list) + byoa_config: list[AgentConfigBinding] = Field(default_factory=list) + tests: list[ReleaseTests] = Field(default_factory=list) + + @model_validator(mode="after") + def explicit_identity(self): + if self.version.lower() == "latest" or "pending" in self.version.lower(): + raise ValueError("agent versions must be explicit") + if self.framework != "code4me2-agent": + if not self.agent_command: + raise ValueError("installed agents need an explicit ACP command") + if self.adapter is None: + raise ValueError("installed agents need an explicit adapter") + from .distributions import missing_gateway_bindings, requires_inference_gateway + + if requires_inference_gateway(self.framework): + missing = missing_gateway_bindings( + {binding.field: binding.model_dump(mode="json") for binding in self.byoa_config} + ) + if missing: + raise ValueError( + f"{self.framework} releases must bind the research inference gateway: " + + ", ".join(missing) + ) + elif self.agent_command or self.agent_command_args or self.byoa_config: + raise ValueError("the managed runtime uses its packaged --managed entrypoint") + if len({(test.os, test.arch) for test in self.tests}) != len(self.tests): + raise ValueError("duplicate test platform") + return self + + +class RuntimeInput(BaseModel): + model_config = ConfigDict(extra="forbid") + manifest: str + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + +class ParticipantRecipe(BaseModel): + model_config = ConfigDict(extra="forbid") + schema_version: Literal["1"] = "1" + plugin_version: str = Field(pattern=SEMVER) + plugin_commit: str = Field(pattern=r"^[0-9a-f]{40}$") + server_commit: str = Field(pattern=r"^[0-9a-f]{40}$") + runtime: RuntimeInput + agents: list[AgentInput] + # Existing profile request fields only; populated with derived release pins. + profiles: list[dict[str, Any]] = Field(default_factory=list) + + @model_validator(mode="after") + def agent_choices(self): + frameworks = [agent.framework for agent in self.agents] + if frameworks.count("code4me2-agent") != 1 or len(frameworks) != len(set(frameworks)): + raise ValueError("the recipe must declare one managed agent and no duplicate agents") + names = set() + allowed = {"name", "model", "framework_version", "connection_id", "tools_json", + "approval_policy", "max_steps", "is_active", "temperature", "max_context_tokens"} + required = {"name", "model", "framework_version", "connection_id", "approval_policy", "max_steps"} + for profile in self.profiles: + if set(profile) - allowed or required - set(profile): + raise ValueError("profile templates must use explicit non-secret profile request fields") + if profile["framework_version"] not in frameworks or not profile["name"] or profile["name"] in names: + raise ValueError("profile frameworks must match the recipe and names must be unique") + names.add(profile["name"]) + UUID(str(profile["connection_id"])) + if not isinstance(profile["max_steps"], int) or isinstance(profile["max_steps"], bool) or profile["max_steps"] < 1: + raise ValueError("profile max_steps must be a positive integer") + if profile["approval_policy"] not in {"auto", "per_step", "suggestion_only"}: + raise ValueError("profile approval_policy is unsupported") + if not isinstance(profile["model"], str) or not profile["model"].strip(): + raise ValueError("profile model must be explicit") + tools = json.loads(profile.get("tools_json", "[]")) + if not isinstance(tools, list) or not all(isinstance(t, str) for t in tools): + raise ValueError("profile tools_json must be a string array") + framework = profile["framework_version"] + if framework != "code4me2-agent": + agent = next(a for a in self.agents if a.framework == framework) + required_bindings = {"model", "max_steps", "approval_policy"} + if tools: + required_bindings.add("tools") + if profile.get("temperature") is not None: + required_bindings.add("temperature") + if required_bindings - {b.field for b in agent.byoa_config}: + raise ValueError("installed-agent profile requires missing configuration bindings") + return self + + +def prepare( + recipe: ParticipantRecipe, + inputs: Path, + output: Path, + platforms: Sequence[str] = PLATFORMS, +) -> dict: + """Prepare one recipe document in a new directory. + + ``platforms`` defaults to every supported native platform. A caller may + request an explicit subset for a local development release; the emitted + recipe then declares exactly that subset, so a partial preparation is + visible in every downstream artifact. + """ + requested = tuple(canonical_platform(p) for p in platforms) + if not requested or len(set(requested)) != len(requested): + raise ValueError("platforms must be a non-empty unique subset of the supported native platforms") + + runtime_manifest = input_path(inputs, recipe.runtime.manifest) + if file_sha256(runtime_manifest) != recipe.runtime.sha256: + raise ValueError("runtime release manifest checksum mismatch") + runtime = json.loads(runtime_manifest.read_text(encoding="utf-8")) + managed = next(a for a in recipe.agents if a.framework == "code4me2-agent") + if runtime.get("server_commit") != recipe.server_commit: + raise ValueError("runtime server commit does not match the recipe") + if runtime.get("runtime_version") != managed.version or runtime.get("managed_protocol_version") != "1": + raise ValueError("runtime version/protocol does not match the managed agent") + + expected: dict[str, str] = {} + for projected in runtime.get("artifacts", []): + platform = canonical_platform(f"{projected.get('platform')}-{projected.get('architecture')}") + expected[str(projected.get("archive"))] = platform + if len(expected) != len(requested) or set(expected.values()) != set(requested): + if set(requested) == set(PLATFORMS): + raise ValueError("runtime release must cover all four supported native platforms") + raise ValueError( + "runtime release must cover exactly the requested native platforms: " + + ", ".join(requested) + ) + + artifacts: list[dict[str, Any]] = [] + resources = output / "resources" / "code4me-runtime" + resources.mkdir(parents=True) + for projected in runtime.get("artifacts", []): + archive = str(projected.get("archive")) + source = input_path(runtime_manifest.parent, archive) + digest = file_sha256(source) + if digest != projected.get("sha256"): + raise ValueError(f"runtime archive checksum mismatch: {archive}") + size = source.stat().st_size + if int(projected.get("size") or size) != size: + raise ValueError(f"runtime archive size mismatch: {archive}") + os_name, arch = expected[archive].split("-") + shutil.copyfile(source, resources / Path(archive).name) + artifact_document = { + "runtime_id": str(projected.get("runtime_id") or "code4me-agent"), + "version": managed.version, + "platform": os_name, + "architecture": arch, + "archive": Path(archive).name, + "sha256": digest, + "size": size, + "executable": str( + projected.get("executable") + or ("code4me2-agent.exe" if os_name == "windows" else "code4me2-agent") + ), + "managed_protocol": "1", + "tests": projected.get("tests"), + } + if projected.get("download_url") is not None: + # A published runtime's immutable asset URL must survive the + # preparation recipe; the plugin needs it for assigned studies. + artifact_document["download_url"] = projected["download_url"] + if managed.adapter is not None: + # The plugin verifies the bootstrap's adapter pin per artifact. + artifact_document["adapter"] = managed.adapter.model_dump(mode="json") + artifacts.append(artifact_document) + + agents: list[dict[str, Any]] = [] + for agent in recipe.agents: + if agent.framework == "code4me2-agent": + continue + agents.append({ + "framework": agent.framework, + "version": agent.version, + "adapter": agent.adapter.model_dump(mode="json"), + "agent_command": agent.agent_command, + "agent_command_args": agent.agent_command_args, + "byoa_config": [b.model_dump(mode="json") for b in agent.byoa_config], + "tests": [t.model_dump(mode="json") for t in agent.tests], + }) + + profiles = [] + for profile in recipe.profiles: + profiles.append(dict( + {"tools_json": "[]", "is_active": True, "temperature": None, "max_context_tokens": None}, + **profile, + )) + + document = { + "manifest_version": 1, + "schema_version": "1", + "runtime_version": managed.version, + "managed_protocol_version": "1", + "plugin_version": recipe.plugin_version, + "plugin_commit": recipe.plugin_commit, + "server_commit": recipe.server_commit, + "artifacts": artifacts, + "agents": agents, + "profiles": profiles, + } + if managed.adapter is not None: + document["adapter"] = managed.adapter.model_dump(mode="json") + from .manifest_import import build_manifest_release + + build_manifest_release(document, archives={a["archive"]: resources / a["archive"] for a in artifacts}) + document["recipe_digest"] = "sha256:" + hashlib.sha256( + json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() + ).hexdigest() + write_json(output / "recipe.json", document) + # The participant plugin build copies ``resources/code4me-runtime`` into the + # plugin and its runtime installer reads ``code4me-runtime/manifest.json`` + # with archive paths relative to the resource root: the same recipe, in the + # plugin's spelling. + write_json(resources / "manifest.json", runtime_manifest_for_plugin(document)) + write_json(output / "prepared-inputs.json", { + file.relative_to(output).as_posix(): file_sha256(file) + for file in sorted(output.rglob("*")) if file.is_file() + }) + return document + + +def runtime_manifest_for_plugin(document: dict) -> dict: + """The prepared recipe as the plugin's bundled ``code4me-runtime/manifest.json``.""" + manifest = { + "manifest_version": 1, + "runtime_version": document["runtime_version"], + "managed_protocol_version": document["managed_protocol_version"], + "server_commit": document.get("server_commit"), + "plugin_commit": document.get("plugin_commit"), + "artifacts": [ + {**artifact, "archive": f"code4me-runtime/{artifact['archive']}"} + for artifact in document["artifacts"] + ], + } + if "adapter" in document: + manifest["adapter"] = document["adapter"] + return manifest + +def load_prepared(output: Path) -> dict: + """Catch changed/missing prepared inputs before building or any API write.""" + hashes = json.loads((output / "prepared-inputs.json").read_text()) + if "recipe.json" not in hashes: + raise ValueError("incomplete preparation input inventory") + for name, expected in hashes.items(): + if file_sha256(input_path(output, name)) != expected: + raise ValueError(f"prepared input changed: {name}; prepare into a new directory") + return json.loads((output / "recipe.json").read_text()) + + +def main() -> None: + """The native producer and CI aggregation share the server import contract.""" + import argparse + import platform + import subprocess + import sys + import tempfile + import zipfile + from datetime import datetime, timezone + + from .manifest_import import build_manifest_release + + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + native = commands.add_parser("native", help="build, test and emit one native manifest") + native.add_argument("--version", required=True) + native.add_argument("--platform", required=True) + native.add_argument("--server-commit", required=True) + native.add_argument("--bundle", type=Path, default=Path("dist/code4me2-agent")) + native.add_argument("--output", type=Path, required=True) + native.add_argument("--skip-build", action="store_true", help="use an already built/signed bundle") + native.add_argument("--release-base-url", help="exact public GitHub release URL containing these assets") + native.add_argument("--min-plugin-version") + merge = commands.add_parser("merge", help="verify and combine native CI artifacts") + merge.add_argument("--directory", type=Path, required=True) + merge.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + if args.command == "merge": + documents = [json.loads(path.read_text()) for path in sorted(args.directory.glob("native-*.json"))] + if not documents: + raise ValueError("no native manifests") + document = dict(documents[0]) + for item in documents: + if any(item[key] != document[key] for key in ("runtime_version", "server_commit", "managed_protocol_version")): + raise ValueError("native manifests describe different builds") + document["artifacts"] = [artifact for item in documents for artifact in item["artifacts"]] + if {f"{a['platform']}-{a['architecture']}" for a in document["artifacts"]} != set(PLATFORMS): + raise ValueError("published runtime must cover all four native platforms") + archives = {a["archive"]: input_path(args.directory, a["archive"]) for a in document["artifacts"]} + if set(archives) != {p.name for p in args.directory.glob("*.zip")}: + raise ValueError("archives must exactly match the manifest") + build_manifest_release(document, archives=archives) + write_json(args.output, document) + else: + target = canonical_platform(args.platform) + host = canonical_platform(f"{platform.system()}-{platform.machine()}") + if target != host: + raise ValueError(f"native tests require {target}, current host is {host}") + if not re.fullmatch(SEMVER, args.version) or not re.fullmatch(r"[0-9a-f]{40}", args.server_commit): + raise ValueError("explicit semantic version and full server commit are required") + if args.output.exists(): + raise ValueError("use a new output directory for an immutable release") + if not args.skip_build: + subprocess.run([sys.executable, "packaging/stamp_runtime_version.py", "--version", args.version], check=True, stdout=sys.stderr) + subprocess.run([sys.executable, "-m", "pytest", "-q", "tests/code4me2_agent"], check=True, stdout=sys.stderr) + subprocess.run([sys.executable, "-m", "PyInstaller", "--clean", "--noconfirm", "packaging/code4me2-agent.spec"], check=True, stdout=sys.stderr) + args.output.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="code4me-native-", dir=args.output.parent) as temporary: + stage = Path(temporary) + archive = stage / f"code4me-agent-{target}.zip" + subprocess.run([sys.executable, "packaging/archive_runtime.py", "--root", str(args.bundle), "--platform", target, "--output", str(archive)], check=True, stdout=sys.stderr) + extracted = stage / "runtime" + with zipfile.ZipFile(archive) as bundle: + bundle.extractall(extracted) + executable = "code4me2-agent.exe" if target.startswith("windows") else "code4me2-agent" + binary = (extracted / executable).resolve() + binary.chmod(0o755) + version = subprocess.run([str(binary), "--version"], check=True, capture_output=True, text=True, timeout=60).stdout.strip() + if version != args.version: + raise ValueError("packaged executable version does not match the release") + checked = subprocess.run([str(binary), "--self-check"], check=True, capture_output=True, text=True, timeout=60) + if json.loads(checked.stdout).get("status") != "ok": + raise ValueError("packaged self-check failed") + request = json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": 1, "clientCapabilities": {}, "clientInfo": {"name": "code4me-release-test", "version": "1"}}}) + "\n" + process = subprocess.Popen([str(binary), "--managed"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=extracted) + try: + stdout, stderr = process.communicate(request, timeout=60) + except subprocess.TimeoutExpired: + process.kill() + process.communicate() + raise ValueError("packaged ACP initialize timed out") from None + responses = [json.loads(line) for line in stdout.splitlines() if line.strip()] + response = next((item for item in responses if item.get("id") == 1), {}) + if process.returncode or response.get("result", {}).get("protocolVersion") != 1 or "error" in response: + raise ValueError("packaged ACP initialize failed") + os_name, arch = target.split("-") + document = { + "manifest_version": 1, + "runtime_version": args.version, + "managed_protocol_version": "1", + "server_commit": args.server_commit, + "min_protocol_version": "1", + "max_protocol_version": "1", + "artifacts": [{ + "runtime_id": "code4me-agent", "version": args.version, + "platform": os_name, "architecture": arch, + "archive": archive.name, "sha256": file_sha256(archive), + "size": archive.stat().st_size, "executable": executable, + "tests": {"self_check": "PASS", "acp_initialize": "PASS", "ran_at": datetime.now(timezone.utc).isoformat()}, + }], + } + if args.release_base_url: + document["artifacts"][0]["download_url"] = args.release_base_url.rstrip("/") + "/" + archive.name + if args.min_plugin_version: + document["min_plugin_version"] = args.min_plugin_version + build_manifest_release(document, archives={archive.name: archive}) + write_json(stage / f"native-{target}.json", document) + shutil.rmtree(extracted) + stage.rename(args.output) + print(json.dumps(document)) + + +if __name__ == "__main__": + main() diff --git a/src/research/study/agents/registry.py b/src/research/study/agents/registry.py new file mode 100644 index 0000000..ddbb859 --- /dev/null +++ b/src/research/study/agents/registry.py @@ -0,0 +1,642 @@ +"""Pure agent-registry service: releases, qualification and capability evidence. + +This module has no database or application dependency. It operates on the +Pydantic contracts in :mod:`research.study.agents.models` and returns typed results so +an API can render field-level errors and an audit can record why an operation +was blocked. + +Key invariants: + +* A release's identity is ``(agent_id, release_id, source_manifest_digest)``. A + changed digest requires a distinct release record; an exact duplicate is + rejected with ``DUPLICATE_RELEASE``. +* Qualification is derived from per-platform producer tests. + A terminal operator disable overrides passing tests. +* Platform resolution is exact and never falls back to another platform. +* Declared and observed capabilities are stored in separate maps and unknown or + unavailable measurements stay visible (``value`` stays ``None``). +""" + +from __future__ import annotations + +import re +import uuid +from datetime import datetime, timezone +from typing import Any, Mapping, Optional, Union +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import ( + CapabilityCoverageState, + DistributionMode, + QualificationStatus, + RegistryReasonCode, + SnapshotCapabilityState, +) +from .models import ( + AdapterRef, + AgentReleaseV1, + CapabilityCoverage, + CapabilityCoverageReport, + CapabilityEntry, + CapabilitySnapshotV1, + DistributionArtifact, + EnvironmentRef, + SnapshotFailure, + normalize_platform, +) + +_BASE_CONFIG = ConfigDict(extra="forbid") + +# Qualification states that may be selected by a study condition. +SELECTABLE_STATUSES = frozenset({QualificationStatus.QUALIFIED}) + +EntryInput = Union[CapabilityEntry, SnapshotCapabilityState, str, None] + + +class RegistryIssue(BaseModel): + """One typed registry rejection reason.""" + + model_config = _BASE_CONFIG + + code: RegistryReasonCode + message: str + field: str = "" + + +class RegistrationResult(BaseModel): + """Outcome of registering a release.""" + + model_config = _BASE_CONFIG + + accepted: bool + release: Optional[AgentReleaseV1] = None + issue: Optional[RegistryIssue] = None + + +class ArtifactResolution(BaseModel): + """Outcome of selecting a distribution artifact for a platform.""" + + model_config = _BASE_CONFIG + + resolved: bool + artifact: Optional[DistributionArtifact] = None + issue: Optional[RegistryIssue] = None + + +class DistributionResolution(BaseModel): + """Outcome of resolving the distribution contract for one platform. + + A ``PACKAGED`` release resolves to a digest-pinned [artifact]; a + ``BYOA_EXTERNAL`` release resolves to the participant-installable + command/package identity and carries no artifact. + """ + + model_config = _BASE_CONFIG + + resolved: bool + distribution_mode: DistributionMode = DistributionMode.PACKAGED + artifact: Optional[DistributionArtifact] = None + agent_command: Optional[str] = None + agent_command_args: list[str] = Field(default_factory=list) + agent_package: Optional[str] = None + issue: Optional[RegistryIssue] = None + + +class QualificationAssessment(BaseModel): + """Whether a release currently satisfies the qualification contract.""" + + model_config = _BASE_CONFIG + + qualifiable: bool + blockers: list[RegistryIssue] = Field(default_factory=list) + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _issue(code: RegistryReasonCode, message: str, field: str = "") -> RegistryIssue: + return RegistryIssue(code=code, message=message, field=field) + + +def _distribution_issue(release: AgentReleaseV1) -> Optional[RegistryIssue]: + """Validate the mode-specific distribution contract, or ``None`` if valid. + + ``PACKAGED`` requires at least one artifact with a non-empty digest; + ``BYOA_EXTERNAL`` requires a command/package identity and is *not* expected + to carry a digest. The check is shared by registration and qualification so + a release can never be accepted under one contract and qualified under another. + """ + if release.is_byoa: + if release.byoa_identity is None: + return _issue( + RegistryReasonCode.AGENT_NOT_FOUND, + "a BYOA release must declare an agent command or an agent package", + "agent_command", + ) + return None + + if not release.artifacts: + return _issue( + RegistryReasonCode.ARTIFACT_MISSING, + "a PACKAGED release must publish at least one distribution artifact", + "artifacts", + ) + for index, artifact in enumerate(release.artifacts): + if not artifact.sha256.strip(): + return _issue( + RegistryReasonCode.DIGEST_MISMATCH, + "a PACKAGED distribution artifact must carry a sha256 digest", + f"artifacts[{index}].sha256", + ) + return None + + +def derive_qualification_status(release_json: Optional[Mapping[str, Any]]) -> QualificationStatus: + """Usability comes from complete platform test results; disable is terminal.""" + from pydantic import ValidationError + + from .models import ReleaseTests + + if not isinstance(release_json, Mapping): + return QualificationStatus.UNQUALIFIED + if release_json.get("qualification_status") in {"DISABLED", "BLOCKED", "RETIRED"}: + return QualificationStatus(release_json["qualification_status"]) + try: + tests = [ReleaseTests.model_validate(item) for item in release_json.get("tests", [])] + except (ValidationError, TypeError): + return QualificationStatus.UNQUALIFIED + platforms = {(item.os, item.arch) for item in tests} + if not platforms or len(platforms) != len(tests): + return QualificationStatus.UNQUALIFIED + if release_json.get("distribution_mode") != "BYOA_EXTERNAL": + artifacts = release_json.get("artifacts") or [] + expected = {normalize_platform(a.get("os", ""), a.get("arch", "")) for a in artifacts} + if not expected or not platforms <= expected: + return QualificationStatus.UNQUALIFIED + return QualificationStatus.QUALIFIED + + +# --------------------------------------------------------------------------- +# Approval policy vocabulary (phase 04; folded from approvals.py) +# --------------------------------------------------------------------------- +# +# An agent profile selects one of a small, closed set of approval policies. The +# ``auto``/``per_step``/``suggestion_only`` vocabulary has one owner: +# ``backend.routers.agent.profiles`` validates the same values. With the +# conformance-receipt machinery retired there is no per-release evidence to +# narrow the set, so every supported option is selectable. + +APPROVAL_AUTO = "auto" +APPROVAL_PER_STEP = "per_step" +APPROVAL_SUGGESTION_ONLY = "suggestion_only" + +ALL_APPROVAL_OPTIONS: tuple[str, ...] = ( + APPROVAL_AUTO, + APPROVAL_PER_STEP, + APPROVAL_SUGGESTION_ONLY, +) + + +def verified_approval_options(release_json: Optional[Mapping[str, Any]] = None) -> list[str]: + """Return the approval options selectable for a release. + + Kept as a release-facing helper (the profile editor and catalogue read it), + but it no longer inspects conformance evidence: every supported option is + available. + """ + return list(ALL_APPROVAL_OPTIONS) + + +def approval_option_verified( + release_json: Optional[Mapping[str, Any]], option: str +) -> bool: + """Whether ``option`` is a supported approval policy.""" + return (option or "").strip().lower() in ALL_APPROVAL_OPTIONS + + +def _coerce_entry(value: EntryInput, default_state: SnapshotCapabilityState) -> CapabilityEntry: + """Normalize a capability map value into a :class:`CapabilityEntry`. + + ``None`` becomes an entry with the default state and ``value=None``: a + missing measurement is never turned into ``0`` or ``False``. + """ + if isinstance(value, CapabilityEntry): + return value + if isinstance(value, SnapshotCapabilityState): + return CapabilityEntry(state=value, value=None) + if isinstance(value, str): + try: + state = SnapshotCapabilityState(value.strip().upper()) + except ValueError: + state = SnapshotCapabilityState.UNKNOWN + return CapabilityEntry(state=state, value=None) + if value is None: + return CapabilityEntry(state=default_state, value=None) + return CapabilityEntry(state=default_state, value=value) + + +def build_capability_snapshot( + snapshot_id: UUID, + release: AgentReleaseV1, + *, + protocol_version: str, + declared: Mapping[str, EntryInput], + observed: Mapping[str, EntryInput], + environment: Optional[EnvironmentRef] = None, + adapter_id: Optional[str] = None, + adapter_version: Optional[str] = None, + evidence_refs: Optional[list[str]] = None, + captured_at: Optional[datetime] = None, + failure: Optional[SnapshotFailure] = None, +) -> CapabilitySnapshotV1: + """Build an immutable snapshot from a declared/observed observation pair. + + The two maps are preserved independently: a capability present only in + ``declared`` is not synthesized into ``observed``, and vice versa. + """ + return CapabilitySnapshotV1( + snapshot_id=snapshot_id, + release_id=release.release_id, + agent_id=release.agent_id, + adapter_id=adapter_id or (release.adapter.adapter_id if release.adapter else None), + adapter_version=adapter_version + or (release.adapter.version if release.adapter else None), + environment=environment or EnvironmentRef(), + protocol_version=protocol_version, + declared={ + capability: _coerce_entry(value, SnapshotCapabilityState.DECLARED) + for capability, value in declared.items() + }, + observed={ + capability: _coerce_entry(value, SnapshotCapabilityState.UNKNOWN) + for capability, value in observed.items() + }, + evidence_refs=list(evidence_refs or []), + captured_at=captured_at or _now(), + failure=failure, + ) + + +def _coverage_state( + declared_state: SnapshotCapabilityState, observed_state: SnapshotCapabilityState +) -> CapabilityCoverageState: + if ( + declared_state == SnapshotCapabilityState.BROKEN + or observed_state == SnapshotCapabilityState.BROKEN + ): + return CapabilityCoverageState.BROKEN + if observed_state == SnapshotCapabilityState.PARTIAL: + return CapabilityCoverageState.PARTIAL + if observed_state == SnapshotCapabilityState.UNAVAILABLE: + return CapabilityCoverageState.UNAVAILABLE + if observed_state == SnapshotCapabilityState.OBSERVED: + return CapabilityCoverageState.OBSERVED + if declared_state in ( + SnapshotCapabilityState.DECLARED, + SnapshotCapabilityState.OBSERVED, + ): + return CapabilityCoverageState.DECLARED_ONLY + return CapabilityCoverageState.UNKNOWN + + +def capability_coverage(snapshot: CapabilitySnapshotV1) -> list[CapabilityCoverage]: + """Summarize declared vs observed state per capability (never a boolean).""" + capabilities = sorted(set(snapshot.declared) | set(snapshot.observed)) + entries: list[CapabilityCoverage] = [] + for capability in capabilities: + declared = snapshot.declared.get(capability) + observed = snapshot.observed.get(capability) + declared_state = ( + declared.state if declared else SnapshotCapabilityState.UNKNOWN + ) + observed_state = ( + observed.state if observed else SnapshotCapabilityState.UNKNOWN + ) + limitations: list[str] = [] + if declared: + limitations.extend(declared.limitations) + if observed: + limitations.extend(observed.limitations) + entries.append( + CapabilityCoverage( + capability=capability, + declared_state=declared_state, + observed_state=observed_state, + coverage=_coverage_state(declared_state, observed_state), + value_present=observed is not None and observed.value is not None, + limitations=limitations, + ) + ) + return entries + + +def coverage_report(snapshot: CapabilitySnapshotV1) -> CapabilityCoverageReport: + """Build the coverage report (entries + explicit state counts).""" + entries = capability_coverage(snapshot) + counts: dict[str, int] = {} + for entry in entries: + counts[entry.coverage.value] = counts.get(entry.coverage.value, 0) + 1 + return CapabilityCoverageReport( + snapshot_id=snapshot.snapshot_id, + release_id=snapshot.release_id, + entries=entries, + counts=counts, + ) + + +class AgentRegistry: + """In-memory registry of releases and capability snapshots. + + The service is deliberately storage-agnostic. The API layer loads rows from + the database into a registry instance (or wraps it in + :class:`research.study.agents.resolver.RegistryReleaseResolver`) and persists the + results through :mod:`research.study.agents.store`. + """ + + def __init__(self) -> None: + self._by_identity: dict[tuple[str, str, str], AgentReleaseV1] = {} + self._by_release_id: dict[tuple[str, str], AgentReleaseV1] = {} + self._by_version: dict[tuple[str, str], list[AgentReleaseV1]] = {} + self._snapshots: dict[UUID, CapabilitySnapshotV1] = {} + + # -- releases --------------------------------------------------------- + + def register_release(self, release: AgentReleaseV1) -> RegistrationResult: + """Register a release, rejecting duplicate digest identities.""" + if not release.source_manifest_digest.strip(): + return RegistrationResult( + accepted=False, + issue=_issue( + RegistryReasonCode.DIGEST_MISMATCH, + "source_manifest_digest must be non-empty", + "source_manifest_digest", + ), + ) + + distribution_issue = _distribution_issue(release) + if distribution_issue is not None: + return RegistrationResult(accepted=False, issue=distribution_issue) + + identity = release.digest_identity + if identity in self._by_identity: + return RegistrationResult( + accepted=False, + issue=_issue( + RegistryReasonCode.DUPLICATE_RELEASE, + "an identical release digest already exists", + "source_manifest_digest", + ), + ) + + key = (release.agent_id, release.release_id) + existing = self._by_release_id.get(key) + if existing is not None: + return RegistrationResult( + accepted=False, + issue=_issue( + RegistryReasonCode.DUPLICATE_RELEASE, + ( + "release_id is already registered for this agent with a " + "different digest; a changed digest requires a new release_id" + ), + "release_id", + ), + ) + + stored = release.model_copy(deep=True) + self._by_identity[identity] = stored + self._by_release_id[key] = stored + self._by_version.setdefault((release.agent_id, release.version), []).append( + stored + ) + return RegistrationResult(accepted=True, release=stored.model_copy(deep=True)) + + def get_release( + self, agent_id: str, release_id: str + ) -> Optional[AgentReleaseV1]: + """Fetch a release by its natural ``(agent_id, release_id)`` key.""" + return self._by_release_id.get((agent_id, release_id)) + + def find_release( + self, + agent_id: str, + *, + release_id: Optional[str] = None, + version: Optional[str] = None, + ) -> Optional[AgentReleaseV1]: + """Resolve a release by explicit id, or by a unique version label.""" + if release_id: + return self._by_release_id.get((agent_id, release_id)) + if version: + candidates = self._by_version.get((agent_id, version), []) + if len(candidates) == 1: + return candidates[0] + return None + + def version_candidates( + self, agent_id: str, version: str + ) -> list[AgentReleaseV1]: + """All releases registered for ``(agent_id, version)``.""" + return list(self._by_version.get((agent_id, version), [])) + + def list_releases(self, agent_id: Optional[str] = None) -> list[AgentReleaseV1]: + """List releases, optionally filtered by agent id, in a stable order.""" + releases = list(self._by_release_id.values()) + if agent_id is not None: + releases = [release for release in releases if release.agent_id == agent_id] + return sorted( + releases, + key=lambda release: (release.agent_id, release.version, release.release_id), + ) + + # -- platform selection / qualification ------------------------------ + + def resolve_artifact( + self, release: AgentReleaseV1, os_name: str, arch: str + ) -> ArtifactResolution: + """Return the exact artifact for ``(os_name, arch)`` with no fallback.""" + if not release.artifacts: + return ArtifactResolution( + resolved=False, + issue=_issue( + RegistryReasonCode.ARTIFACT_MISSING, + "release publishes no distribution artifacts", + "artifacts", + ), + ) + + artifact = release.artifact_for(os_name, arch) + if artifact is None: + return ArtifactResolution( + resolved=False, + issue=_issue( + RegistryReasonCode.UNSUPPORTED_PLATFORM, + f"no artifact for platform ({os_name}, {arch})", + "artifacts", + ), + ) + if not artifact.sha256.strip(): + return ArtifactResolution( + resolved=False, + issue=_issue( + RegistryReasonCode.DIGEST_MISMATCH, + f"artifact for ({os_name}, {arch}) has no digest", + "artifacts", + ), + ) + return ArtifactResolution(resolved=True, artifact=artifact) + + def resolve_distribution( + self, release: AgentReleaseV1, os_name: str, arch: str + ) -> DistributionResolution: + """Resolve the mode-specific distribution contract with no fallback. + + A ``BYOA_EXTERNAL`` release resolves to its command/package identity (no + artifact); a ``PACKAGED`` release delegates to :meth:`resolve_artifact` so + a missing/undigested artifact still blocks. + """ + if release.is_byoa: + if release.byoa_identity is None: + return DistributionResolution( + resolved=False, + distribution_mode=DistributionMode.BYOA_EXTERNAL, + issue=_issue( + RegistryReasonCode.AGENT_NOT_FOUND, + "BYOA release declares neither a command nor an agent package", + "agent_command", + ), + ) + return DistributionResolution( + resolved=True, + distribution_mode=DistributionMode.BYOA_EXTERNAL, + agent_command=release.agent_command, + agent_command_args=list(release.agent_command_args), + agent_package=release.agent_package, + ) + + artifact_resolution = self.resolve_artifact(release, os_name, arch) + if not artifact_resolution.resolved: + return DistributionResolution( + resolved=False, + distribution_mode=DistributionMode.PACKAGED, + issue=artifact_resolution.issue, + ) + return DistributionResolution( + resolved=True, + distribution_mode=DistributionMode.PACKAGED, + artifact=artifact_resolution.artifact, + ) + + def assess_qualification( + self, release: AgentReleaseV1 + ) -> QualificationAssessment: + """Return the blockers preventing a release from being qualified.""" + blockers: list[RegistryIssue] = [] + + distribution_issue = _distribution_issue(release) + if distribution_issue is not None: + blockers.append(distribution_issue) + elif not release.is_byoa: + for index, artifact in enumerate(release.artifacts): + if not artifact.path.strip(): + blockers.append( + _issue( + RegistryReasonCode.ARTIFACT_MISSING, + "distribution artifact path must be non-empty", + f"artifacts[{index}].path", + ) + ) + + # The adapter is optional release metadata: a manifest that declares no + # adapter identity keeps ``adapter=None`` and is never gated on it. + adapter: Optional[AdapterRef] = release.adapter + if adapter is not None and not adapter.version.strip(): + blockers.append( + _issue( + RegistryReasonCode.ADAPTER_INCOMPATIBLE, + "adapter version must be non-empty", + "adapter.version", + ) + ) + + minimum = release.min_protocol_version + maximum = release.max_protocol_version + if minimum and maximum and _version_tuple(minimum) > _version_tuple(maximum): + blockers.append( + _issue( + RegistryReasonCode.INVALID_VERSION_RANGE, + f"max_protocol_version {maximum!r} is below {minimum!r}", + "max_protocol_version", + ) + ) + + return QualificationAssessment(qualifiable=not blockers, blockers=blockers) + + # -- snapshots -------------------------------------------------------- + + def record_snapshot( + self, + release: AgentReleaseV1, + *, + protocol_version: str, + declared: Mapping[str, EntryInput], + observed: Mapping[str, EntryInput], + environment: Optional[EnvironmentRef] = None, + adapter_id: Optional[str] = None, + adapter_version: Optional[str] = None, + evidence_refs: Optional[list[str]] = None, + captured_at: Optional[datetime] = None, + failure: Optional[SnapshotFailure] = None, + ) -> CapabilitySnapshotV1: + """Build and store an immutable capability snapshot for ``release``.""" + snapshot = build_capability_snapshot( + uuid.uuid4(), + release, + protocol_version=protocol_version, + declared=declared, + observed=observed, + environment=environment, + adapter_id=adapter_id, + adapter_version=adapter_version, + evidence_refs=evidence_refs, + captured_at=captured_at, + failure=failure, + ) + self.add_snapshot(snapshot) + return snapshot + + def add_snapshot(self, snapshot: CapabilitySnapshotV1) -> CapabilitySnapshotV1: + """Store an already-built snapshot (e.g. rehydrated from storage).""" + self._snapshots[snapshot.snapshot_id] = snapshot + return snapshot + + def get_snapshot(self, snapshot_id: UUID) -> Optional[CapabilitySnapshotV1]: + """Fetch a snapshot by id, or ``None``.""" + return self._snapshots.get(snapshot_id) + + def list_snapshots( + self, release_id: Optional[str] = None + ) -> list[CapabilitySnapshotV1]: + """List snapshots, optionally for one release, oldest-first.""" + snapshots = list(self._snapshots.values()) + if release_id is not None: + snapshots = [ + snapshot + for snapshot in snapshots + if snapshot.release_id == release_id + ] + return sorted(snapshots, key=lambda snapshot: snapshot.captured_at) + + def coverage(self, snapshot_id: UUID) -> Optional[CapabilityCoverageReport]: + """Coverage report for one stored snapshot, or ``None`` if unknown.""" + snapshot = self.get_snapshot(snapshot_id) + if snapshot is None: + return None + return coverage_report(snapshot) + + +def _version_tuple(value: str) -> tuple[int, ...]: + return tuple(int(part) for part in re.findall(r"\d+", value)) diff --git a/src/research/study/agents/resolver.py b/src/research/study/agents/resolver.py new file mode 100644 index 0000000..396f363 --- /dev/null +++ b/src/research/study/agents/resolver.py @@ -0,0 +1,176 @@ +"""Bridge from the agent registry to protocol release validation. + +:class:`RegistryReleaseResolver` implements the +:class:`research.study.protocol.validation.ReleaseResolver` interface so protocol +publication rejects releases that are missing, retired, blocked, or not yet +qualified: + +* not found -> ``NOT_FOUND`` +* ``RETIRED`` / ``BLOCKED`` / ``DISABLED`` -> ``WITHDRAWN`` +* ``UNQUALIFIED`` / ``DRAFT`` / ``CONDITIONALLY_QUALIFIED`` -> ``UNQUALIFIED`` +* ``QUALIFIED`` -> ``RESOLVED`` with the selected artifact's digest +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +from research.study.protocol.enums import ReleaseResolutionStatus +from research.study.protocol.validation import ReleaseResolution + +from .enums import QualificationStatus + +if TYPE_CHECKING: + from .models import AgentReleaseV1 + from .registry import AgentRegistry + +_WITHDRAWN = frozenset( + { + QualificationStatus.RETIRED, + QualificationStatus.BLOCKED, + QualificationStatus.DISABLED, + } +) +_UNQUALIFIED = frozenset( + { + QualificationStatus.UNQUALIFIED, + QualificationStatus.DRAFT, + QualificationStatus.CONDITIONALLY_QUALIFIED, + } +) + + +class RegistryReleaseResolver: + """Resolve a study condition's pinned release against the registry.""" + + def __init__( + self, + registry: AgentRegistry, + *, + platform: Optional[tuple[str, str]] = None, + ) -> None: + self.registry = registry + self.platform = platform + + def resolve( + self, + agent_id: str, + *, + release_id: Optional[str] = None, + version: Optional[str] = None, + ) -> ReleaseResolution: + """Map a release reference onto a typed protocol resolution.""" + if version is not None and version.strip().lower() == "latest": + return ReleaseResolution( + status=ReleaseResolutionStatus.UNQUALIFIED, + agent_id=agent_id, + release_id=release_id, + version=version, + message="'latest' is a mutable reference and cannot be pinned", + ) + + release = self.registry.find_release( + agent_id, release_id=release_id, version=version + ) + if release is None: + if version is not None and self.registry.version_candidates( + agent_id, version + ): + return ReleaseResolution( + status=ReleaseResolutionStatus.UNQUALIFIED, + agent_id=agent_id, + release_id=release_id, + version=version, + message=( + "version label is ambiguous; pin an explicit release_id" + ), + ) + return ReleaseResolution( + status=ReleaseResolutionStatus.NOT_FOUND, + agent_id=agent_id, + release_id=release_id, + version=version, + message="release is not registered", + ) + + if release.qualification_status in _WITHDRAWN: + return ReleaseResolution( + status=ReleaseResolutionStatus.WITHDRAWN, + agent_id=agent_id, + release_id=release.release_id, + version=release.version, + message=f"release is {release.qualification_status.value}", + ) + + if release.qualification_status in _UNQUALIFIED: + return ReleaseResolution( + status=ReleaseResolutionStatus.UNQUALIFIED, + agent_id=agent_id, + release_id=release.release_id, + version=release.version, + message=( + "release is not qualified " + f"({release.qualification_status.value})" + ), + ) + + if release.is_byoa: + # A BYOA release is pinned by its command/package identity, not by an + # artifact digest; there is nothing to stage or verify up front. + if release.byoa_identity is None: + return ReleaseResolution( + status=ReleaseResolutionStatus.UNQUALIFIED, + agent_id=agent_id, + release_id=release.release_id, + version=release.version, + distribution_mode=release.distribution_mode.value, + message=( + "qualified BYOA release has no command or package identity" + ), + ) + return ReleaseResolution( + status=ReleaseResolutionStatus.RESOLVED, + agent_id=agent_id, + release_id=release.release_id, + version=release.version, + distribution_mode=release.distribution_mode.value, + ) + + digest = self.artifact_digest(release) + if not digest: + return ReleaseResolution( + status=ReleaseResolutionStatus.UNQUALIFIED, + agent_id=agent_id, + release_id=release.release_id, + version=release.version, + message="qualified release has no resolvable artifact digest", + ) + + return ReleaseResolution( + status=ReleaseResolutionStatus.RESOLVED, + agent_id=agent_id, + release_id=release.release_id, + version=release.version, + artifact_digest=digest, + distribution_mode=release.distribution_mode.value, + ) + + def artifact_digest(self, release: AgentReleaseV1) -> Optional[str]: + """Return the digest of the selected artifact, or ``None``. + + When a platform is configured it is selected exactly; otherwise the + release must publish exactly one artifact, or the first artifact in a + stable ``(os, arch)`` order is used. + """ + if self.platform is not None: + resolution = self.registry.resolve_artifact( + release, self.platform[0], self.platform[1] + ) + if not resolution.resolved or resolution.artifact is None: + return None + return resolution.artifact.sha256 + + artifacts = sorted(release.artifacts, key=lambda item: (item.os, item.arch)) + if not artifacts: + return None + return artifacts[0].sha256 diff --git a/src/research/study/agents/store.py b/src/research/study/agents/store.py new file mode 100644 index 0000000..dc5a7ab --- /dev/null +++ b/src/research/study/agents/store.py @@ -0,0 +1,192 @@ +"""CRUD-style persistence helpers for the agent registry. + +These functions take a caller-managed SQLAlchemy ``Session`` so the core package +never imports ``App`` or touches the application singleton. The router is +responsible for session lifecycle (``App.get_db_session`` / ``rollback`` / +``close``). +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional, Sequence + +from sqlalchemy import select + +if TYPE_CHECKING: + import uuid + + from sqlalchemy.orm import Session + + from .models import AgentReleaseV1, CapabilitySnapshotV1 + +from database.research_schemas import AgentRelease, ResearchAgentRun + +from .enums import QualificationStatus +from .registry import derive_qualification_status + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def upsert_release(session: Session, release: AgentReleaseV1, *, commit: bool = True) -> AgentRelease: + """Persist verified import results without overwriting an existing release.""" + row = session.get(AgentRelease, release.release_id) + if row is not None: + if (row.agent_id, row.source_manifest_digest) != (release.agent_id, release.source_manifest_digest): + raise ValueError("release identity is immutable") + return row + payload = release.model_dump(mode="json") + status = derive_qualification_status(payload) + payload["qualification_status"] = status.value + row = AgentRelease( + release_id=release.release_id, + agent_id=release.agent_id, + source_manifest_digest=release.source_manifest_digest, + status=status.value, + release_json=payload, + created_at=release.created_at or _now(), + ) + session.add(row) + if commit: + session.commit() + session.refresh(row) + return row + + +def get_release(session: Session, release_id: str) -> Optional[AgentRelease]: + """Fetch a release row by primary key, or ``None``.""" + return session.get(AgentRelease, release_id) + + +def list_releases( + session: Session, agent_id: Optional[str] = None +) -> Sequence[AgentRelease]: + """List release rows (optionally for one agent) in a stable order.""" + statement = select(AgentRelease) + if agent_id is not None: + statement = statement.where(AgentRelease.agent_id == agent_id) + statement = statement.order_by( + AgentRelease.agent_id.asc(), AgentRelease.release_id.asc() + ) + return list(session.execute(statement).scalars().all()) + + +def insert_snapshot(session: Session, snapshot: CapabilitySnapshotV1) -> ResearchAgentRun: + """Store a capability snapshot on the run that captured it. + + A snapshot is a property of an agent run, so it is written to + ``research_agent_run.snapshot_json``. The admin qualification upload has no + participant session, so it records a detached run (``research_session_id`` + is ``None``) keyed by the snapshot id. + """ + payload = snapshot.model_dump(mode="json") + row = session.get(ResearchAgentRun, snapshot.snapshot_id) + if row is None: + row = ResearchAgentRun( + agent_run_id=snapshot.snapshot_id, + research_session_id=None, + agent_release_id=snapshot.release_id, + started_at=snapshot.captured_at, + snapshot_json=payload, + snapshot_captured_at=snapshot.captured_at, + ) + else: + row.snapshot_json = payload + row.snapshot_captured_at = snapshot.captured_at + session.add(row) + session.commit() + session.refresh(row) + return row + + +def get_snapshot( + session: Session, snapshot_id: uuid.UUID +) -> Optional[ResearchAgentRun]: + """Fetch the run holding a snapshot by snapshot id, or ``None``.""" + row = session.get(ResearchAgentRun, snapshot_id) + if row is None or row.snapshot_json is None: + return None + return row + + +def list_snapshots( + session: Session, release_id: Optional[str] = None +) -> Sequence[ResearchAgentRun]: + """List snapshot-bearing runs (optionally for one release), oldest-first.""" + statement = select(ResearchAgentRun).where( + ResearchAgentRun.snapshot_json.is_not(None) + ) + if release_id is not None: + statement = statement.where( + ResearchAgentRun.snapshot_json["release_id"].astext == release_id + ) + statement = statement.order_by(ResearchAgentRun.snapshot_captured_at.asc()) + return list(session.execute(statement).scalars().all()) + + +def row_to_release(row: AgentRelease) -> AgentReleaseV1: + """Rehydrate a stored release row into the versioned model. + + ``release_json`` also carries packaging evidence that is not part of the + release model, so only the model's own fields are validated; the + producer test results determine usability. Terminal operator status wins. + """ + from .models import AgentReleaseV1 as AgentReleaseV1Model + + payload = dict(row.release_json or {}) + model_fields = set(AgentReleaseV1Model.model_fields) + filtered = {key: value for key, value in payload.items() if key in model_fields} + if not isinstance(filtered.get("tests", []), list): + filtered["tests"] = [] # Old aggregate verdicts do not qualify any platform. + release = AgentReleaseV1Model.model_validate(filtered) + if row.status in {"DISABLED", "RETIRED", "BLOCKED"}: + payload["qualification_status"] = row.status + return release.model_copy(update={"qualification_status": derive_qualification_status(payload)}) + + +def row_to_snapshot(row: ResearchAgentRun) -> CapabilitySnapshotV1: + """Rehydrate a snapshot-bearing run row into the versioned model.""" + from .models import CapabilitySnapshotV1 as CapabilitySnapshotV1Model + + return CapabilitySnapshotV1Model.model_validate(row.snapshot_json) + + +def disable_release(session: Session, release_id: str) -> Optional[AgentRelease]: + """Irreversibly disable a release, including subsequent identical imports.""" + row = session.get(AgentRelease, release_id) + if row is None: + return None + row.status = QualificationStatus.DISABLED.value + row.release_json = dict(row.release_json or {}, qualification_status=row.status) + session.commit() + session.refresh(row) + return row + + +def release_summary(row: AgentRelease) -> dict[str, Any]: + release = row_to_release(row) + return { + "release_id": row.release_id, + "agent_id": row.agent_id, + "source_manifest_digest": row.source_manifest_digest, + "status": release.qualification_status.value, + "tests": [item.model_dump(mode="json") for item in release.tests], + "created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else None, + } + + +def snapshot_summary(row: ResearchAgentRun) -> dict[str, Any]: + """Return a compact snapshot summary safe for list responses.""" + payload = row.snapshot_json or {} + captured_at = row.snapshot_captured_at + return { + "snapshot_id": str(payload.get("snapshot_id") or row.agent_run_id), + "release_id": payload.get("release_id"), + "adapter_version": payload.get("adapter_version"), + "protocol_version": payload.get("protocol_version"), + "captured_at": ( + captured_at.isoformat() if isinstance(captured_at, datetime) else None + ), + } diff --git a/src/research/study/consent.py b/src/research/study/consent.py new file mode 100644 index 0000000..9347c89 --- /dev/null +++ b/src/research/study/consent.py @@ -0,0 +1,224 @@ +"""Researcher-authored consent forms and the version each participant accepts. + +A study may freeze a custom consent ``document`` and tick-box ``statements`` in +its configuration (``research_config_json["consent"]``); without one the stock +notice applies with a single required statement. Either way the participant +also sees the platform notice, which states what the platform records (composed +from the frozen telemetry policy by ``backend.routers.research.join``). + +The *view* a participant is shown (document, notice and statements) is digested, +and the enrollment stores that digest plus the view with the participant's +answers, so every enrollment records exactly what it agreed to. The document is +plain text: it is rendered with its line breaks and links, never as HTML. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from typing import Any, Optional + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationError, + field_validator, + model_validator, +) + +from research.canonical import canonical_hash + +__all__ = [ + "CONSENT_VIEW_VERSION", + "DEFAULT_STATEMENT", + "ConsentConfigV1", + "ConsentError", + "ConsentStatementV1", + "accept", + "build_view", + "custom_consent", + "validate_consent_config", + "view_digest", +] + +CONSENT_VIEW_VERSION = 1 +MAX_DOCUMENT_CHARS = 20_000 +MAX_STATEMENTS = 20 +MAX_STATEMENT_CHARS = 500 + +#: The stock form's single statement (the label of the original checkbox). +DEFAULT_STATEMENT: dict[str, Any] = { + "id": "accept", + "text": "I accept the study consent notice.", + "required": True, +} + +_CONTROL_CHARACTERS = re.compile(r"[\x00-\x08\x0b-\x1f\x7f]") + + +def _normalized_text(value: str) -> str: + text = value.replace("\r\n", "\n").replace("\r", "\n").strip() + if _CONTROL_CHARACTERS.search(text): + raise ValueError("must not contain control characters") + return text + + +class ConsentStatementV1(BaseModel): + """One tick-box statement; ``required`` ones must be ticked to join.""" + + model_config = ConfigDict(extra="forbid") + + id: str = Field(pattern=r"^[a-z0-9][a-z0-9_-]{0,31}$") + text: str + required: bool = True + + @field_validator("text") + @classmethod + def _text(cls, value: str) -> str: + text = _normalized_text(value) + if not 1 <= len(text) <= MAX_STATEMENT_CHARS: + raise ValueError(f"must be 1-{MAX_STATEMENT_CHARS} characters") + return text + + +class ConsentConfigV1(BaseModel): + """A study's custom consent form, frozen with its configuration.""" + + model_config = ConfigDict(extra="forbid") + + document: str + statements: list[ConsentStatementV1] = Field(min_length=1, max_length=MAX_STATEMENTS) + + @field_validator("document") + @classmethod + def _document(cls, value: str) -> str: + text = _normalized_text(value) + if not 1 <= len(text) <= MAX_DOCUMENT_CHARS: + raise ValueError(f"must be 1-{MAX_DOCUMENT_CHARS} characters") + return text + + @model_validator(mode="after") + def _statements(self) -> "ConsentConfigV1": + ids = [statement.id for statement in self.statements] + if len(set(ids)) != len(ids): + raise ValueError("statement ids must be unique") + if not any(statement.required for statement in self.statements): + raise ValueError("at least one statement must be required") + return self + + +class ConsentError(Exception): + """A refused consent configuration or acceptance, mapped to an API error.""" + + def __init__( + self, + code: str, + message: str, + *, + status_code: int = 409, + field: Optional[str] = None, + missing: Optional[list[str]] = None, + ) -> None: + super().__init__(message) + self.code = code + self.status_code = status_code + self.field = field + self.missing = missing or [] + + def detail(self) -> dict[str, Any]: + detail: dict[str, Any] = {"code": self.code, "message": str(self)} + if self.field: + detail["field"] = self.field + if self.missing: + detail["missing"] = self.missing + return detail + + +def validate_consent_config(raw: Any) -> dict[str, Any]: + """Validate a create request's custom form; return its normalized JSON.""" + try: + return ConsentConfigV1.model_validate(raw).model_dump(mode="json") + except ValidationError as error: + first = error.errors()[0] + field = ".".join(str(part) for part in first.get("loc", ())) or "consent" + raise ConsentError( + "CONSENT_INVALID", + f"{field}: {first.get('msg', 'invalid consent form')}", + status_code=422, + field=field, + ) from error + + +def custom_consent(config: Any) -> Optional[ConsentConfigV1]: + """The study's frozen custom form, or ``None`` for the stock notice.""" + raw = config.get("consent") if isinstance(config, Mapping) else None + if not raw: + return None + return ConsentConfigV1.model_validate(raw) + + +def build_view(config: Any, notice: str) -> dict[str, Any]: + """What a participant is shown and accepts for one study.""" + custom = custom_consent(config) + return { + "version": CONSENT_VIEW_VERSION, + "document": custom.document if custom is not None else None, + "notice": notice, + "statements": ( + [statement.model_dump(mode="json") for statement in custom.statements] + if custom is not None + else [dict(DEFAULT_STATEMENT)] + ), + } + + +def view_digest(view: Mapping[str, Any]) -> str: + """The version identity of a consent view.""" + return canonical_hash(dict(view)) + + +def accept( + view: Mapping[str, Any], + digest: Optional[str], + answers: Optional[Mapping[str, bool]], +) -> tuple[str, dict[str, Any]]: + """Check one acceptance; return ``(digest, snapshot)`` to store on the enrollment. + + A custom form must be accepted against the digest the participant saw. The + stock form may still be accepted without one (older clients tick a single + box), in which case its one statement counts as ticked. + """ + expected = view_digest(view) + statements = list(view.get("statements") or []) + custom = view.get("document") is not None + if digest is not None and digest != expected: + raise ConsentError("CONSENT_CHANGED", "the consent form changed; review it again") + if digest is None and custom: + raise ConsentError("CONSENT_CHANGED", "review the study's consent form before joining") + given = dict(answers or {}) + if not custom and digest is None and not given: + given = {statement["id"]: True for statement in statements} + unknown = sorted(set(given) - {statement["id"] for statement in statements}) + if unknown: + raise ConsentError( + "CONSENT_STATEMENTS_INVALID", + f"unknown consent statements: {', '.join(unknown)}", + status_code=422, + ) + missing = [ + statement["id"] + for statement in statements + if statement.get("required") and given.get(statement["id"]) is not True + ] + if missing: + raise ConsentError( + "CONSENT_STATEMENTS_REQUIRED", + "tick every required consent statement to join", + missing=missing, + ) + snapshot = { + **dict(view), + "answers": {statement["id"]: given.get(statement["id"]) is True for statement in statements}, + } + return expected, snapshot diff --git a/src/research/study/lifecycle.py b/src/research/study/lifecycle.py new file mode 100644 index 0000000..f5bea60 --- /dev/null +++ b/src/research/study/lifecycle.py @@ -0,0 +1,707 @@ +"""Transactional lifecycle operations for research studies.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from decimal import Decimal +from typing import Any, Optional, Sequence +import uuid +import secrets + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from agents.tools import set_harness_profile_fields +from database.db_schemas import AgentTask, ResearchStudyStatus, Study +from database.research_schemas import ( + RECORD_KIND_STUDY_LIFECYCLE, + InferenceReservation, + ResearchEnrollment, + ResearchEvent, + ResearchRecord, + ResearchSessionV1, + StudyAgentProfile, + StudyAssignment, +) +from research.budget import ledger as budget_ledger +from research.canonical import canonical_hash +from research.participants import identity as identity_store +from research.participants.enums import EnrollmentStatus +from research.participants.models import ResearchEligibility +from research.runtime.assignment.hashing import ( + assignment_strategy, + hashed_profile, + manual_override_enabled, +) +from research.runtime.sessions.enums import CloseReason +from research.study.protocol.enums import AssignmentStrategy +from research.study.protocol.store import build_profile_selections + + +@dataclass(frozen=True) +class StudyStopSummary: + """Counts and identity returned by a terminal study stop.""" + + study_id: uuid.UUID + enrollment_count: int + assignment_count: int + session_count: int + + +@dataclass(frozen=True) +class EnrollmentRevokeSummary: + """Summary returned after revoking one participant enrollment.""" + + enrollment_id: uuid.UUID + session_count: int + + +@dataclass(frozen=True) +class StudyEnrollmentSummary: + """The enrollment and sticky profile assignment created by web consent.""" + + enrollment_id: uuid.UUID + study_id: uuid.UUID + assignment_id: Optional[uuid.UUID] + agent_profile_id: Optional[uuid.UUID] + created: bool + reused: bool + + +class StudyStoppedError(PermissionError): + """Raised when a terminal study cannot accept a new enrollment.""" + + +class ActiveEnrollmentError(PermissionError): + """Raised when an account already has an active enrollment elsewhere.""" + + +class AlreadyEnrolledError(PermissionError): + """Raised when an account attempts to rejoin a terminal enrollment.""" + + +class ResearchStudyNotFoundError(ValueError): + """Raised when a lifecycle operation resolves no research study row.""" + + +class CloneNotAllowedError(PermissionError): + """Raised when a study that is not stopped is asked to clone.""" + + +@dataclass(frozen=True) +class AssignmentOverrideSummary: + """The outcome of a manual arm change (``changed`` is False for a no-op).""" + + enrollment_id: uuid.UUID + changed: bool + from_profile_id: Optional[uuid.UUID] + to_profile_id: uuid.UUID + strategy: str + assigned_at: Optional[datetime] + + +class AssignmentOverrideError(Exception): + """A refused manual arm change; ``code``/``status_code`` map to the API error.""" + + def __init__(self, code: str, message: str, status_code: int = 409) -> None: + super().__init__(message) + self.code = code + self.status_code = status_code + + +def profile_snapshot(profile: Any) -> dict[str, Any]: + """Return the non-secret profile configuration frozen into a study. + + ``system_prompt`` is included only when set, so a profile without one keeps + the snapshot shape (and digest) it had before the field existed; the same + holds for the built-in runtime's ``commands_denylist`` / + ``command_timeout_seconds`` / ``harness_options`` (decision D-01). + """ + snapshot = { + "profile_id": str(profile.profile_id), + "name": profile.name, + "model": profile.model, + "framework_version": profile.framework_version, + "release_id": profile.release_id, + "connection_id": str(profile.connection_id) if profile.connection_id else None, + "tools_json": profile.tools_json, + "approval_policy": profile.approval_policy, + "max_steps": profile.max_steps, + "temperature": profile.temperature, + "max_context_tokens": profile.max_context_tokens, + } + system_prompt = getattr(profile, "system_prompt", None) + if system_prompt is not None: + snapshot["system_prompt"] = system_prompt + snapshot.update(set_harness_profile_fields(profile)) + return snapshot + + +def allocate_join_code(session: Session) -> str: + """Allocate a study-owned join code in the fresh schema.""" + for _ in range(8): + candidate = secrets.token_hex(4).upper() + exists = session.execute( + select(Study.study_id).where(Study.join_code == candidate) + ).scalar_one_or_none() + if exists is None or not isinstance(exists, uuid.UUID): + return candidate + raise RuntimeError("unable to allocate a unique study join code") + + +def update_research_metadata( + session: Session, + study_id: uuid.UUID, + *, + name: Optional[str] = None, + description: Optional[str] = None, + starts_at: Optional[datetime] = None, + ends_at: Optional[datetime] = None, +) -> Study: + """Update allowed metadata before the first consent locks the study.""" + study = session.execute( + select(Study).where(Study.study_id == study_id).with_for_update() + ).scalar_one_or_none() + if study is None or not bool(getattr(study, "is_research", False)): + raise ValueError("research study not found") + if getattr(study, "consent_locked_at", None) is not None: + raise PermissionError("study metadata is locked after first consent") + if getattr(study, "research_status", None) == ResearchStudyStatus.STUDY_STOPPED.value: + raise PermissionError("stopped study cannot be edited") + + if name is not None: + normalized_name = name.strip() + if not normalized_name: + raise ValueError("study name must not be blank") + setattr(study, "name", normalized_name) + if description is not None: + setattr(study, "description", description) + if starts_at is not None: + setattr(study, "starts_at", starts_at) + if ends_at is not None: + setattr(study, "ends_at", ends_at) + session.commit() + session.refresh(study) + return study + + +def open_study_enrollment( + session: Session, + account_id: uuid.UUID, + join_code: str, + *, + now: Optional[datetime] = None, + rng: Optional[Any] = None, + consent_digest: Optional[str] = None, + consent_snapshot: Optional[dict[str, Any]] = None, +) -> StudyEnrollmentSummary: + """Accept web consent and create enrollment/profile assignment atomically. + + ``consent_digest``/``consent_snapshot`` record the consent view the + participant accepted (see :mod:`research.study.consent`). + """ + timestamp = now or datetime.now(timezone.utc) + normalized_code = str(join_code or "").strip().upper() + study = session.execute( + select(Study) + .where(Study.join_code == normalized_code) + .with_for_update() + ).scalar_one_or_none() + if study is None or not bool(getattr(study, "is_research", False)): + raise ValueError("join code not found") + if getattr(study, "research_status", None) == ResearchStudyStatus.STUDY_STOPPED.value: + raise StudyStoppedError("study is stopped") + + participant_row = identity_store.get_or_create_participant_row( + session, account_id, now=timestamp, commit=False + ) + participant_row = identity_store.lock_participant_by_account(session, account_id) or participant_row + active_row = identity_store.get_active_enrollment_for_participant(session, participant_row.participant_id) + if active_row is not None: + if active_row.study_id != study.study_id: + raise ActiveEnrollmentError("account already has an active enrollment") + assignment = session.execute( + select(StudyAssignment).where(StudyAssignment.enrollment_id == active_row.enrollment_id) + ).scalar_one_or_none() + # A reused join changes no state, but it still acquired study and + # participant locks. Close the read transaction so callers that keep + # the session alive cannot retain those locks after idempotent consent. + session.commit() + return StudyEnrollmentSummary( + enrollment_id=active_row.enrollment_id, + study_id=study.study_id, + assignment_id=getattr(assignment, "assignment_id", None), + agent_profile_id=getattr(assignment, "agent_profile_id", None), + created=False, + reused=True, + ) + + prior_row = identity_store.get_enrollment_for_participant_study( + session, participant_row.participant_id, study.study_id + ) + if prior_row is not None: + raise AlreadyEnrolledError("account cannot rejoin this study") + + selected_profiles = list( + session.execute( + select(StudyAgentProfile) + .where(StudyAgentProfile.study_id == study.study_id) + .order_by(StudyAgentProfile.selection_order.asc()) + .with_for_update() + ).scalars().all() + ) + if not selected_profiles: + raise ValueError("study has no selected agent profiles") + enrollment_id = uuid.uuid4() + # The study's frozen assignment policy decides the draw; studies created + # before the policy existed keep the equal-probability CSPRNG pick. + strategy = assignment_strategy(getattr(study, "research_config_json", None)) + if strategy == AssignmentStrategy.DETERMINISTIC_HASH.value: + selected = hashed_profile( + selected_profiles, study_id=study.study_id, enrollment_id=enrollment_id + ) + elif strategy == AssignmentStrategy.RANDOM_EQUAL.value: + selected = rng.choice(selected_profiles) if rng is not None else secrets.choice(selected_profiles) + else: + raise ValueError(f"unknown assignment strategy {strategy!r}") + enrollment = ResearchEnrollment( + enrollment_id=enrollment_id, + participant_id=participant_row.participant_id, + study_id=study.study_id, + participant_code=identity_store.generate_participant_code(), + status=EnrollmentStatus.ACTIVE.value, + revocation_epoch=0, + eligibility_json=ResearchEligibility(eligible=True).model_dump(mode="json"), + enrolled_at=timestamp, + updated_at=timestamp, + consent_accepted_at=timestamp, + consent_digest=consent_digest, + consent_snapshot_json=consent_snapshot, + retention_action="RETAIN_ANONYMIZED", + ) + assignment = StudyAssignment( + assignment_id=uuid.uuid4(), + enrollment_id=enrollment_id, + study_id=study.study_id, + agent_profile_id=selected.profile_id, + strategy=strategy, + randomization_epoch=0, + profile_digest=selected.profile_digest, + profile_snapshot_json=selected.profile_snapshot_json, + status="ACTIVE", + assigned_at=timestamp, + ) + session.add_all([enrollment, assignment]) + # The participant's inference budget is born with the enrollment, in the + # same transaction, at the study's current default (0 = refuse until set). + session.flush() + budget_ledger.create_balance( + session, + enrollment_id=enrollment_id, + study_id=study.study_id, + limit_micro_usd=int(getattr(study, "inference_budget_default_micro_usd", 0) or 0), + now=timestamp, + ) + if getattr(study, "consent_locked_at", None) is None: + setattr(study, "consent_locked_at", timestamp) + setattr(study, "research_status", ResearchStudyStatus.ACTIVE.value) + setattr(study, "is_active", True) + session.commit() + return StudyEnrollmentSummary( + enrollment_id=enrollment_id, + study_id=study.study_id, + assignment_id=assignment.assignment_id, + agent_profile_id=assignment.agent_profile_id, + created=True, + reused=False, + ) + + +def clone_stopped_research_study( + session: Session, + study_id: uuid.UUID, + *, + actor: str, + profile_ids: Optional[Sequence[uuid.UUID]] = None, + allow_shared_profiles: bool = False, + inference_budget_default_micro_usd: Optional[int] = None, +) -> Study: + """Create a fresh DRAFT from stopped metadata without copying participants. + + When ``profile_ids`` is supplied they are validated with the same + owner/active/release-qualified and executable-contract rules as + ``create_study`` and their ``StudyAgentProfile`` rows are inserted in the + clone transaction, so the clone is reachable to a runnable state (ISSUE-12 + Option B). When it is omitted the clone stays profile-less by design; join + then refuses it with "study has no selected agent profiles". + """ + source = session.execute( + select(Study).where(Study.study_id == study_id).with_for_update() + ).scalar_one_or_none() + if source is None or not bool(getattr(source, "is_research", False)): + raise ResearchStudyNotFoundError("research study not found") + if getattr(source, "research_status", None) != ResearchStudyStatus.STUDY_STOPPED.value: + raise CloneNotAllowedError("only stopped studies can be cloned") + + source_config = dict(getattr(source, "research_config_json", None) or {}) + source_config.pop("agent_profile_ids", None) + timestamp = datetime.now(timezone.utc) + selected_profile_ids = list(profile_ids or []) + clone_id = uuid.uuid4() + selections: list[StudyAgentProfile] = [] + if selected_profile_ids: + selections = build_profile_selections( + session, + study_id=clone_id, + profile_ids=selected_profile_ids, + created_by=source.created_by, + allow_shared_profiles=allow_shared_profiles, + timestamp=timestamp, + ) + source_config["profile_ids"] = [str(profile_id) for profile_id in selected_profile_ids] + else: + source_config.pop("profile_ids", None) + clone = Study( + study_id=clone_id, + name=f"{source.name} (copy)", + description=source.description, + created_by=source.created_by, + starts_at=source.starts_at, + ends_at=source.ends_at, + is_active=False, + default_config_id=None, + is_research=True, + research_status=ResearchStudyStatus.DRAFT.value, + research_config_json=source_config, + research_config_digest=canonical_hash(source_config), + join_code=allocate_join_code(session), + created_at=timestamp, + # Participant budgets are copied (or overridden), never reset to zero. + inference_budget_default_micro_usd=( + int(inference_budget_default_micro_usd) + if inference_budget_default_micro_usd is not None + else int(getattr(source, "inference_budget_default_micro_usd", 0) or 0) + ), + inference_budget_warning_fraction=getattr( + source, "inference_budget_warning_fraction", None + ) or Decimal("0.800"), + inference_budget_updated_at=timestamp, + inference_budget_updated_by=actor, + ) + session.add(clone) + for selection in selections: + session.add(selection) + session.commit() + session.refresh(clone) + return clone + + +def enrollments_with_activity( + session: Session, + study_id: uuid.UUID, + enrollment_ids: Optional[Sequence[uuid.UUID]] = None, +) -> set[uuid.UUID]: + """Enrollments of a study that have used their arm. + + Any research session, canonical event, metered inference reservation or agent + task counts: the built-in agent and the inference relay resolve the + assignment without a research session, so sessions alone are not enough. + """ + scope = select(ResearchEnrollment.enrollment_id).where(ResearchEnrollment.study_id == study_id) + if enrollment_ids is not None: + scope = scope.where(ResearchEnrollment.enrollment_id.in_(list(enrollment_ids))) + used: set[uuid.UUID] = set() + for column in ( + ResearchSessionV1.enrollment_id, + ResearchEvent.enrollment_id, + InferenceReservation.enrollment_id, + AgentTask.enrollment_id, + ): + used.update( + session.execute(select(column).where(column.in_(scope)).distinct()).scalars().all() + ) + used.discard(None) + return used + + +def reassign_enrollment( + session: Session, + study_id: uuid.UUID, + enrollment_id: uuid.UUID, + profile_id: uuid.UUID, + *, + actor: Optional[str], + now: Optional[datetime] = None, +) -> AssignmentOverrideSummary: + """Set one participant's arm by hand before they have used it. + + Only studies whose frozen assignment policy allows it, only for an active + enrollment with no activity yet, and only to one of the study's own arms. + The row keeps its ``randomization_epoch``, so the randomized arm stays + recomputable; ``strategy`` becomes ``MANUAL``. The enrollment's revocation + epoch is bumped so any credential minted concurrently is void, and an + audit record names the old and new arm. + """ + timestamp = now or datetime.now(timezone.utc) + # Lock order matches stop/revoke: study, enrollment, assignment. The + # bootstrap of a manual-assignment study reads the assignment under a share + # lock, so a first session and an override serialize on that row; the + # enrollment is locked FOR NO KEY UPDATE so the session insert's foreign-key + # check is never blocked behind it (which would deadlock the two). + study = session.execute( + select(Study).where(Study.study_id == study_id).with_for_update(read=True) + ).scalar_one_or_none() + if study is None or not bool(getattr(study, "is_research", False)): + raise ResearchStudyNotFoundError("research study not found") + if not manual_override_enabled(getattr(study, "research_config_json", None)): + raise AssignmentOverrideError( + "MANUAL_ASSIGNMENT_DISABLED", "this study does not allow manual assignment" + ) + if getattr(study, "research_status", None) == ResearchStudyStatus.STUDY_STOPPED.value: + raise AssignmentOverrideError("STUDY_STOPPED", "the study has been stopped") + enrollment = session.execute( + select(ResearchEnrollment) + .where( + ResearchEnrollment.enrollment_id == enrollment_id, + ResearchEnrollment.study_id == study_id, + ) + .with_for_update(key_share=True) + ).scalar_one_or_none() + if enrollment is None: + raise AssignmentOverrideError( + "ENROLLMENT_NOT_FOUND", "no such enrollment in this study", status_code=404 + ) + if getattr(enrollment, "status", None) != EnrollmentStatus.ACTIVE.value: + raise AssignmentOverrideError("ENROLLMENT_NOT_ACTIVE", "the enrollment is not active") + selection = session.execute( + select(StudyAgentProfile).where( + StudyAgentProfile.study_id == study_id, + StudyAgentProfile.profile_id == profile_id, + ) + ).scalar_one_or_none() + if selection is None: + raise AssignmentOverrideError( + "PROFILE_NOT_IN_STUDY", "the profile is not one of this study's arms", status_code=422 + ) + assignment = session.execute( + select(StudyAssignment) + .where(StudyAssignment.enrollment_id == enrollment_id) + .with_for_update() + ).scalar_one_or_none() + if assignment is None: + raise AssignmentOverrideError("ASSIGNMENT_MISSING", "the enrollment has no assignment yet") + if enrollment_id in enrollments_with_activity(session, study_id, [enrollment_id]): + raise AssignmentOverrideError( + "ASSIGNMENT_IN_USE", "the participant has already used the assigned arm" + ) + + from_profile_id = getattr(assignment, "agent_profile_id", None) + if from_profile_id == profile_id: + # Nothing changes; end the transaction so the locks are released. + session.commit() + return AssignmentOverrideSummary( + enrollment_id=enrollment_id, + changed=False, + from_profile_id=from_profile_id, + to_profile_id=profile_id, + strategy=str(getattr(assignment, "strategy", "")), + assigned_at=getattr(assignment, "assigned_at", None), + ) + + setattr(assignment, "agent_profile_id", selection.profile_id) + setattr(assignment, "profile_digest", selection.profile_digest) + setattr(assignment, "profile_snapshot_json", selection.profile_snapshot_json) + setattr(assignment, "strategy", AssignmentStrategy.MANUAL.value) + setattr(assignment, "assigned_at", timestamp) + setattr(enrollment, "revocation_epoch", int(getattr(enrollment, "revocation_epoch", 0) or 0) + 1) + setattr(enrollment, "updated_at", timestamp) + session.add( + ResearchRecord( + record_id=uuid.uuid4(), + kind=RECORD_KIND_STUDY_LIFECYCLE, + scope_type="enrollment", + scope_id=enrollment_id, + study_id=study_id, + actor=actor, + occurred_at=timestamp, + payload_json={ + "event": "ASSIGNMENT_OVERRIDDEN", + "from_profile_id": str(from_profile_id) if from_profile_id else None, + "to_profile_id": str(selection.profile_id), + }, + ) + ) + session.commit() + return AssignmentOverrideSummary( + enrollment_id=enrollment_id, + changed=True, + from_profile_id=from_profile_id, + to_profile_id=selection.profile_id, + strategy=AssignmentStrategy.MANUAL.value, + assigned_at=timestamp, + ) + + +def revoke_research_enrollment( + session: Session, + study_id: uuid.UUID, + enrollment_id: uuid.UUID, + *, + now: Optional[datetime] = None, +) -> EnrollmentRevokeSummary: + """Revoke one enrollment while retaining all research records.""" + enrollment = session.execute( + select(ResearchEnrollment) + .where( + ResearchEnrollment.enrollment_id == enrollment_id, + ResearchEnrollment.study_id == study_id, + ) + .with_for_update() + ).scalar_one_or_none() + if enrollment is None: + raise ValueError("enrollment not found") + timestamp = now or datetime.now(timezone.utc) + if getattr(enrollment, "status", None) != "REVOKED": + setattr(enrollment, "status", "REVOKED") + setattr( + enrollment, + "revocation_epoch", + int(getattr(enrollment, "revocation_epoch", 0)) + 1, + ) + setattr(enrollment, "updated_at", timestamp) + assignments = list( + session.execute( + select(StudyAssignment) + .where(StudyAssignment.enrollment_id == enrollment_id) + .with_for_update() + ) + .scalars() + .all() + ) + for assignment in assignments: + setattr(assignment, "status", "REVOKED") + sessions = list( + session.execute( + select(ResearchSessionV1) + .where( + ResearchSessionV1.enrollment_id == enrollment_id, + ResearchSessionV1.state.not_in(("ended", "revoked")), + ) + .with_for_update() + ) + .scalars() + .all() + ) + for research_session in sessions: + setattr(research_session, "state", "revoked") + setattr(research_session, "closed_at", timestamp) + setattr(research_session, "close_reason", CloseReason.REVOKED.value) + session.commit() + return EnrollmentRevokeSummary( + enrollment_id=enrollment_id, + session_count=len(sessions), + ) + + +def stop_research_study( + session: Session, + study_id: uuid.UUID, + *, + actor: str, + now: Optional[datetime] = None, +) -> StudyStopSummary: + """Stop a research study without deleting retained research data.""" + study = session.execute( + select(Study).where(Study.study_id == study_id).with_for_update() + ).scalar_one_or_none() + if study is None or not bool(getattr(study, "is_research", False)): + raise ValueError("research study not found") + + timestamp = now or datetime.now(timezone.utc) + was_active = getattr(study, "research_status", None) != ResearchStudyStatus.STUDY_STOPPED.value + if was_active: + setattr(study, "research_status", ResearchStudyStatus.STUDY_STOPPED.value) + setattr(study, "is_active", False) + setattr(study, "stopped_at", timestamp) + setattr(study, "stopped_by", actor) + + enrollments = list( + session.execute( + select(ResearchEnrollment) + .where( + ResearchEnrollment.study_id == study_id, + ResearchEnrollment.status == "ACTIVE", + ) + .with_for_update() + ) + .scalars() + .all() + ) + enrollment_ids = [enrollment.enrollment_id for enrollment in enrollments] + for enrollment in enrollments: + setattr(enrollment, "status", "STUDY_STOPPED") + setattr( + enrollment, + "revocation_epoch", + int(getattr(enrollment, "revocation_epoch", 0)) + 1, + ) + setattr(enrollment, "updated_at", timestamp) + + assignments = [] + if enrollment_ids: + assignments = list( + session.execute( + select(StudyAssignment) + .where(StudyAssignment.enrollment_id.in_(enrollment_ids)) + .with_for_update() + ) + .scalars() + .all() + ) + for assignment in assignments: + setattr(assignment, "status", "STUDY_STOPPED") + + sessions = list( + session.execute( + select(ResearchSessionV1) + .where( + ResearchSessionV1.study_id == study_id, + ResearchSessionV1.state.not_in(("ended", "revoked")), + ) + .with_for_update() + ) + .scalars() + .all() + ) + for research_session in sessions: + setattr(research_session, "state", "revoked") + setattr(research_session, "closed_at", timestamp) + setattr(research_session, "close_reason", "STUDY_STOPPED") + + if was_active: + session.add( + ResearchRecord( + record_id=uuid.uuid4(), + kind=RECORD_KIND_STUDY_LIFECYCLE, + scope_type="study", + scope_id=study_id, + study_id=study_id, + actor=actor, + occurred_at=timestamp, + payload_json={ + "event": "STUDY_STOPPED", + "enrollment_count": len(enrollments), + "assignment_count": len(assignments), + "session_count": len(sessions), + }, + ) + ) + + session.commit() + return StudyStopSummary( + study_id=study_id, + enrollment_count=len(enrollments), + assignment_count=len(assignments), + session_count=len(sessions), + ) diff --git a/src/research/study/packaging/__init__.py b/src/research/study/packaging/__init__.py new file mode 100644 index 0000000..26da212 --- /dev/null +++ b/src/research/study/packaging/__init__.py @@ -0,0 +1,71 @@ +"""Runtime packaging (Issue 11). + +Public surface: + +* :mod:`research.study.packaging.enums` - typed package reason codes. +* :mod:`research.study.packaging.models` - ``RuntimeManifestV2`` and its + digest-addressed components. +* :mod:`research.study.packaging.verifier` - default-deny package verification and + path containment. +* :mod:`research.study.packaging.resolver` - exact platform selection with a required + bootstrap-digest match and no PATH/package-manager fallback. +* :mod:`research.study.packaging.packager` - synthetic package assembly helper. +* :mod:`research.study.packaging.store` - Session-supplied persistence helpers. + +The core package never imports ``App``, FastAPI, or a session factory. +Release usability comes from platform tests owned by :mod:`research.study.agents`. +""" + +from .enums import PackageReasonCode +from .models import ( + ComponentEntry, + PackageIssue, + PackageVerificationResult, + PlatformTriple, + ProtocolCompatibility, + ResolutionResult, + ResolvedComponent, + RuntimeManifestV2, + SelfCheckSpec, + manifest_digest_of, + normalize_sha256, + sha256_of_bytes, +) +from .packager import build_package, write_manifest +from .resolver import resolve_component, resolved_digest +from .verifier import ( + PathContainment, + hash_file, + resolve_under_root, + scan_secret_files, + scan_undeclared_executables, + sign_manifest, + verify_package, +) + +__all__ = [ + "ComponentEntry", + "PackageIssue", + "PackageReasonCode", + "PackageVerificationResult", + "PathContainment", + "PlatformTriple", + "ProtocolCompatibility", + "ResolutionResult", + "ResolvedComponent", + "RuntimeManifestV2", + "SelfCheckSpec", + "build_package", + "hash_file", + "manifest_digest_of", + "normalize_sha256", + "resolve_component", + "resolve_under_root", + "resolved_digest", + "scan_secret_files", + "scan_undeclared_executables", + "sha256_of_bytes", + "sign_manifest", + "verify_package", + "write_manifest", +] diff --git a/src/research/study/packaging/enums.py b/src/research/study/packaging/enums.py new file mode 100644 index 0000000..942a653 --- /dev/null +++ b/src/research/study/packaging/enums.py @@ -0,0 +1,52 @@ +"""Closed vocabularies for runtime packaging (Issue 11). + +Every member is part of a persisted or reported contract (package manifests, +verification results). Members are additive only. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = ["PackageReasonCode"] + + +class PackageReasonCode(str, Enum): + """Stable machine-readable reason a package/component was blocked.""" + + OK = "OK" + + # Manifest structure. + INVALID_MANIFEST = "INVALID_MANIFEST" + SCHEMA_VERSION_UNSUPPORTED = "SCHEMA_VERSION_UNSUPPORTED" + EMPTY_COMPONENTS = "EMPTY_COMPONENTS" + DUPLICATE_PLATFORM = "DUPLICATE_PLATFORM" + + # Path containment. + PATH_ESCAPE = "PATH_ESCAPE" + PATH_UNSAFE = "PATH_UNSAFE" + + # Integrity. + DIGEST_MISMATCH = "DIGEST_MISMATCH" + SIZE_MISMATCH = "SIZE_MISMATCH" + SIGNATURE_MISMATCH = "SIGNATURE_MISMATCH" + ARTIFACT_MISSING = "ARTIFACT_MISSING" + MISSING_COMPONENT = "MISSING_COMPONENT" + NOT_EXECUTABLE = "NOT_EXECUTABLE" + + # Archive hygiene. + UNDECLARED_EXECUTABLE = "UNDECLARED_EXECUTABLE" + SECRET_FILE_PRESENT = "SECRET_FILE_PRESENT" + + # Selection. + UNSUPPORTED_PLATFORM = "UNSUPPORTED_PLATFORM" + BOOTSTRAP_DIGEST_MISMATCH = "BOOTSTRAP_DIGEST_MISMATCH" + + # Self-check / tooling. + SELF_CHECK_FAILED = "SELF_CHECK_FAILED" + WRITER_UNAVAILABLE = "WRITER_UNAVAILABLE" + + # Qualification. + RECIPE_TESTS_FAILED = "RECIPE_TESTS_FAILED" + RELEASE_DISABLED = "RELEASE_DISABLED" + RELEASE_NOT_FOUND = "RELEASE_NOT_FOUND" diff --git a/src/research/study/packaging/models.py b/src/research/study/packaging/models.py new file mode 100644 index 0000000..3de3447 --- /dev/null +++ b/src/research/study/packaging/models.py @@ -0,0 +1,288 @@ +"""Pydantic v2 contracts for runtime packaging (Issue 11). + +``RuntimeManifestV2`` is authoritative for the exact contents and integrity of a +participant runtime package. + +Digest convention: component payload digests and the manifest digest are +``sha256:<64 lowercase hex>`` so they compare directly with the agent +``DistributionArtifact.sha256`` and the bootstrap ``artifact_digest``. A bare hex +digest is also accepted when comparing (see :func:`normalize_sha256`). +""" + +from __future__ import annotations + +import hashlib +from typing import Optional + +from pydantic import BaseModel, ConfigDict, Field + +from research.canonical import canonical_hash +from research.study.agents.models import AdapterRef + +from .enums import ( + PackageReasonCode, # noqa: TC001 - pydantic resolves annotations at runtime +) + +_BASE = ConfigDict(extra="forbid") + +__all__ = [ + "AdapterRef", + "ComponentEntry", + "PackageIssue", + "PackageVerificationResult", + "PlatformTriple", + "ProtocolCompatibility", + "ResolutionResult", + "ResolvedComponent", + "RuntimeManifestV2", + "SelfCheckSpec", + "manifest_digest_of", + "normalize_sha256", + "sha256_of_bytes", +] + + +def sha256_of_bytes(data: bytes) -> str: + """Return ``sha256:`` for ``data``.""" + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def normalize_sha256(value: Optional[str]) -> Optional[str]: + """Return the bare lowercase hex of ``value`` (stripping ``sha256:``), or ``None``.""" + if value is None: + return None + text = value.strip().lower() + if text.startswith("sha256:"): + text = text[len("sha256:") :] + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + return None + return text + + +class PlatformTriple(BaseModel): + """An ``(os, arch)`` selection key.""" + + model_config = _BASE + + os: str + arch: str + + @property + def key(self) -> str: + """The ``os-arch`` tuple used in diagnostics and args templates.""" + return f"{self.os}-{self.arch}" + + +class ComponentEntry(BaseModel): + """One platform-specific component payload inside a package.""" + + model_config = _BASE + + name: str + os: str + arch: str + relative_path: str + sha256: str + size: int + executable: bool = False + license: Optional[str] = None + + @property + def platform(self) -> PlatformTriple: + """The component's platform triple.""" + return PlatformTriple(os=self.os, arch=self.arch) + + +class SelfCheckSpec(BaseModel): + """A declared self-check command (never a shell string).""" + + model_config = _BASE + + command: list[str] = Field(default_factory=list) + expected_exit_code: int = 0 + + +class ProtocolCompatibility(BaseModel): + """ACP protocol compatibility range for the packaged runtime.""" + + model_config = _BASE + + min_protocol_version: Optional[str] = None + max_protocol_version: Optional[str] = None + + def supports(self, protocol_version: Optional[str]) -> Optional[bool]: + """Whether ``protocol_version`` is in range; ``None`` when undecidable.""" + if protocol_version is None: + return None + if self.min_protocol_version is None and self.max_protocol_version is None: + return None + current = _version_tuple(protocol_version) + if current is None: + return None + if self.min_protocol_version is not None: + minimum = _version_tuple(self.min_protocol_version) + if minimum is not None and current < minimum: + return False + if self.max_protocol_version is not None: + maximum = _version_tuple(self.max_protocol_version) + if maximum is not None and current > maximum: + return False + return True + + +def _version_tuple(value: str) -> Optional[tuple[int, ...]]: + try: + parts = [int(part) for part in value.strip().split(".")] + except (TypeError, ValueError): + return None + return tuple(parts) if parts else None + + +class RuntimeManifestV2(BaseModel): + """The package manifest: authoritative for contents and integrity.""" + + model_config = _BASE + + schema_version: str = "2" + manifest_digest: str = "" + release_id: str + agent_id: str + adapter_ref: AdapterRef + components: list[ComponentEntry] = Field(default_factory=list) + supported_platforms: list[PlatformTriple] = Field(default_factory=list) + # Argument template keyed by platform key ("macos-arm64"); always an array. + args_template: dict[str, list[str]] = Field(default_factory=dict) + licenses: list[str] = Field(default_factory=list) + compatibility: ProtocolCompatibility = Field(default_factory=ProtocolCompatibility) + self_check: Optional[SelfCheckSpec] = None + signature: Optional[str] = None + + def platform_keys(self) -> list[str]: + """Supported platform keys in declared order.""" + return [platform.key for platform in self.supported_platforms] + + def supports_platform(self, os: str, arch: str) -> bool: + """Whether ``(os, arch)`` is a declared supported platform.""" + return any( + platform.os == os and platform.arch == arch + for platform in self.supported_platforms + ) + + def component_for(self, os: str, arch: str) -> Optional[ComponentEntry]: + """The first component whose ``(os, arch)`` exactly matches, or ``None``.""" + return next( + ( + component + for component in self.components + if component.os == os and component.arch == arch + ), + None, + ) + + def executable_component_for(self, os: str, arch: str) -> Optional[ComponentEntry]: + """The runnable component for ``(os, arch)``, or the first, or ``None``.""" + matches = [ + component + for component in self.components + if component.os == os and component.arch == arch + ] + return next((c for c in matches if c.executable), matches[0] if matches else None) + + def args_for_platform(self, os: str, arch: str) -> list[str]: + """Argument-array template for a platform (falls back to ``os`` then empty).""" + key = f"{os}-{arch}" + if key in self.args_template: + return list(self.args_template[key]) + if os in self.args_template: + return list(self.args_template[os]) + return [] + + def canonical_payload(self) -> dict: + """Manifest fields except ``manifest_digest`` and ``signature``. + + Both the digest and the HMAC are computed over this payload, so a signed + manifest can also be digest-verified without a circular dependency. + """ + payload = self.model_dump(mode="json") + payload.pop("manifest_digest", None) + payload.pop("signature", None) + return payload + + def computed_digest(self) -> str: + """``sha256:`` over the canonical payload.""" + return "sha256:" + canonical_hash(self.canonical_payload()) + + def digest_matches(self) -> bool: + """Whether the stored digest matches the recomputed payload digest.""" + stored = normalize_sha256(self.manifest_digest) + return stored is not None and stored == normalize_sha256(self.computed_digest()) + + def with_digest(self) -> "RuntimeManifestV2": + """Return a copy with ``manifest_digest`` set to the computed value.""" + return self.model_copy(update={"manifest_digest": self.computed_digest()}) + + def platform_conflicts(self) -> list[str]: + """Platform keys declared by more than one component.""" + seen: dict[str, int] = {} + for component in self.components: + seen[component.platform.key] = seen.get(component.platform.key, 0) + 1 + return sorted(key for key, count in seen.items() if count > 1) + + def supports_protocol(self, protocol_version: Optional[str]) -> Optional[bool]: + """Whether the packaged runtime supports ``protocol_version``.""" + return self.compatibility.supports(protocol_version) + + +def manifest_digest_of(payload: dict) -> str: + """Compute the manifest digest for a raw payload dict (``manifest_digest``/``signature`` removed).""" + trimmed = dict(payload) + trimmed.pop("manifest_digest", None) + trimmed.pop("signature", None) + return "sha256:" + canonical_hash(trimmed) + + +class PackageIssue(BaseModel): + """One typed package/verification finding.""" + + model_config = _BASE + + code: PackageReasonCode + message: str + field: str = "" + + +class PackageVerificationResult(BaseModel): + """Typed aggregate result of verifying a package directory.""" + + model_config = _BASE + + valid: bool + manifest_digest: Optional[str] = None + verified_components: int = 0 + errors: list[PackageIssue] = Field(default_factory=list) + + @property + def codes(self) -> set[PackageReasonCode]: + """The set of reason codes observed.""" + return {error.code for error in self.errors} + + +class ResolvedComponent(BaseModel): + """A component selected for one exact platform and bootstrap digest.""" + + model_config = _BASE + + component: ComponentEntry + os: str + arch: str + arguments: list[str] = Field(default_factory=list) + relative_path: str + + +class ResolutionResult(BaseModel): + """Typed result of resolving a component with no fallback.""" + + model_config = _BASE + + resolved: Optional[ResolvedComponent] = None + error: Optional[PackageIssue] = None diff --git a/src/research/study/packaging/packager.py b/src/research/study/packaging/packager.py new file mode 100644 index 0000000..a91fc76 --- /dev/null +++ b/src/research/study/packaging/packager.py @@ -0,0 +1,97 @@ +"""Synthetic package assembly helper (Issue 11). + +``build_package`` writes a directory of files and returns a fully-populated, +digest-correct :class:`RuntimeManifestV2`. It exists so tests (and a future CI +builder) can assemble a package without any network or host state. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Mapping, Optional, Sequence + +from .models import ( + ComponentEntry, + PlatformTriple, + ProtocolCompatibility, + RuntimeManifestV2, + SelfCheckSpec, + sha256_of_bytes, +) +from .verifier import sign_manifest + +if TYPE_CHECKING: + from pathlib import Path + + from research.study.agents.models import AdapterRef + +__all__ = ["build_package", "write_manifest"] + + +def build_package( + root: Path, + *, + release_id: str, + agent_id: str, + adapter_ref: AdapterRef, + os: str, + arch: str, + files: Mapping[str, bytes], + executable: Optional[str] = None, + component_name: str = "runtime", + args_template: Optional[Sequence[str]] = None, + licenses: Optional[Sequence[str]] = None, + compatibility: Optional[ProtocolCompatibility] = None, + self_check: Optional[SelfCheckSpec] = None, + sign_secret: Optional[str] = None, +) -> RuntimeManifestV2: + """Write ``files`` under ``root`` and return a digest-correct manifest.""" + root.mkdir(parents=True, exist_ok=True) + for relative, content in files.items(): + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + if relative == executable: + target.chmod(target.stat().st_mode | 0o111) + + if executable is not None and executable not in files: + raise ValueError(f"executable '{executable}' is not one of the package files") + + # One component per payload file keeps every declared path exact. + components = [ + ComponentEntry( + name=component_name if relative == (executable or "") else f"{component_name}:{relative}", + os=os, + arch=arch, + relative_path=relative, + sha256=sha256_of_bytes(content), + size=len(content), + executable=(relative == executable), + ) + for relative, content in files.items() + ] + + manifest = RuntimeManifestV2( + schema_version="2", + release_id=release_id, + agent_id=agent_id, + adapter_ref=adapter_ref, + components=components, + supported_platforms=[PlatformTriple(os=os, arch=arch)], + args_template={f"{os}-{arch}": list(args_template or [])}, + licenses=list(licenses or []), + compatibility=compatibility or ProtocolCompatibility(), + self_check=self_check, + ) + if sign_secret: + # Sign the unsigned payload, then digest the signed manifest. + manifest = manifest.model_copy(update={"signature": sign_manifest(manifest, sign_secret)}) + return manifest.with_digest() + + +def write_manifest(path: Path, manifest: RuntimeManifestV2) -> Path: + """Write a manifest as canonical JSON to ``path``.""" + from research.canonical import canonical_json + + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(canonical_json(manifest.model_dump(mode="json")), encoding="utf-8") + return path diff --git a/src/research/study/packaging/resolver.py b/src/research/study/packaging/resolver.py new file mode 100644 index 0000000..ba5ac77 --- /dev/null +++ b/src/research/study/packaging/resolver.py @@ -0,0 +1,95 @@ +"""Exact runtime component selection (Issue 11). + +``resolve_component`` selects the component for one exact ``(os, arch)`` and +requires its digest to equal the digest the bootstrap manifest declared. There is +deliberately **no fallback**: no ``PATH`` lookup, no ``npm``/``npx``, no source +checkout, no other platform or version. A failure returns a typed block and no +component, so a caller can never launch an unaudited executable. +""" + +from __future__ import annotations + +from typing import Optional + +from .enums import PackageReasonCode +from .models import ( + PackageIssue, + ResolutionResult, + ResolvedComponent, + RuntimeManifestV2, + normalize_sha256, +) + +__all__ = ["resolve_component", "resolved_digest"] + + +def resolve_component( + manifest: RuntimeManifestV2, + os: str, + arch: str, + bootstrap_artifact_digest: str, +) -> ResolutionResult: + """Resolve the exact component, or return a typed block (never a fallback).""" + if not os or not arch: + return ResolutionResult( + error=PackageIssue( + code=PackageReasonCode.UNSUPPORTED_PLATFORM, + message="a non-blank os and arch are required to resolve a runtime", + ) + ) + if not manifest.supports_platform(os, arch): + return ResolutionResult( + error=PackageIssue( + code=PackageReasonCode.UNSUPPORTED_PLATFORM, + message=f"manifest does not declare platform '{os}-{arch}'", + field="supported_platforms", + ) + ) + component = manifest.executable_component_for(os, arch) + if component is None: + return ResolutionResult( + error=PackageIssue( + code=PackageReasonCode.MISSING_COMPONENT, + message=f"manifest has no component for '{os}-{arch}'", + field="components", + ) + ) + + declared = normalize_sha256(component.sha256) + if declared is None: + return ResolutionResult( + error=PackageIssue( + code=PackageReasonCode.DIGEST_MISMATCH, + message=f"component '{component.name}' has a malformed sha256", + field="components.sha256", + ) + ) + expected = normalize_sha256(bootstrap_artifact_digest) + if expected is None or expected != declared: + return ResolutionResult( + error=PackageIssue( + code=PackageReasonCode.BOOTSTRAP_DIGEST_MISMATCH, + message=( + f"component '{component.name}' digest does not match the " + "bootstrap-declared artifact digest" + ), + field="artifact_digest", + ) + ) + + return ResolutionResult( + resolved=ResolvedComponent( + component=component, + os=os, + arch=arch, + arguments=manifest.args_for_platform(os, arch), + relative_path=component.relative_path, + ) + ) + + +def resolved_digest(result: ResolutionResult) -> Optional[str]: + """Convenience: the resolved component digest, or ``None`` when blocked.""" + if result.resolved is None: + return None + return normalize_sha256(result.resolved.component.sha256) diff --git a/src/research/study/packaging/store.py b/src/research/study/packaging/store.py new file mode 100644 index 0000000..6b65667 --- /dev/null +++ b/src/research/study/packaging/store.py @@ -0,0 +1,173 @@ +"""Persistence helpers for runtime packages (Issue 11). + +Packaging evidence is not a table: a release's ``RuntimeManifestV2`` lives in +``agent_release.release_json.package_json``. Every helper takes a caller-supplied +SQLAlchemy ``Session`` and works in domain models (:class:`RuntimeManifestV2`), +never raw ORM rows. The packaging package itself imports no FastAPI/App. + +Recording package metadata never changes release test results or status. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional, Sequence + +from sqlalchemy import select + +from database.research_schemas import AgentRelease + +from .models import RuntimeManifestV2 + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + +__all__ = [ + "PackageView", + "get_package", + "get_package_by_digest", + "list_packages", + "package_summary", + "row_to_manifest", + "upsert_package", +] + +#: ``release_json`` keys that carry packaging evidence. +PACKAGE_KEY = "package_json" + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +@dataclass(frozen=True) +class PackageView: + """One release's stored runtime package (a projection of ``release_json``).""" + + package_id: uuid.UUID + release_id: str + os: Optional[str] + arch: Optional[str] + manifest_digest: str + manifest_json: dict[str, Any] + signature: Optional[str] + created_at: Optional[datetime] + + +def _package_id(manifest_digest: str) -> uuid.UUID: + return uuid.uuid5( + uuid.NAMESPACE_URL, f"code4me2://runtime-package/{manifest_digest}" + ) + + +def _package_from_release(row: AgentRelease) -> Optional[PackageView]: + data = row.release_json or {} + manifest = data.get(PACKAGE_KEY) + if not isinstance(manifest, dict): + return None + platforms = manifest.get("supported_platforms") or [] + platform = platforms[0] if platforms else {} + manifest_digest = str(manifest.get("manifest_digest") or "") + return PackageView( + package_id=_package_id(manifest_digest), + release_id=row.release_id, + os=platform.get("os"), + arch=platform.get("arch"), + manifest_digest=manifest_digest, + manifest_json=manifest, + signature=manifest.get("signature"), + created_at=row.created_at, + ) + + +def _iter_packages(session: Session) -> list[PackageView]: + statement = select(AgentRelease) + views = [ + _package_from_release(row) + for row in session.execute(statement).scalars().all() + ] + return [view for view in views if view is not None] + + +def upsert_package( + session: Session, + manifest: RuntimeManifestV2, + *, + package_id: Optional[uuid.UUID] = None, +) -> PackageView: + """Store a release's package manifest, or return the existing digest view. + + The package is folded into ``agent_release.release_json.package_json`` (the + release must already have been imported). ``package_id`` is derived from + the manifest digest. Recording metadata cannot create or qualify a release. + """ + existing = get_package_by_digest(session, manifest.manifest_digest) + if existing is not None: + return existing + row = session.get(AgentRelease, manifest.release_id) + if row is None: + raise ValueError("import the verified release before registering package metadata") + data = dict(row.release_json or {}) + data[PACKAGE_KEY] = manifest.model_dump(mode="json") + row.release_json = data + session.commit() + session.refresh(row) + view = _package_from_release(row) + assert view is not None + return view + + +def get_package(session: Session, package_id: uuid.UUID) -> Optional[PackageView]: + """Fetch a package view by its deterministic package id, or ``None``.""" + for view in _iter_packages(session): + if view.package_id == package_id: + return view + return None + + +def get_package_by_digest( + session: Session, manifest_digest: str +) -> Optional[PackageView]: + """Fetch a package view by its manifest digest, or ``None``.""" + for view in _iter_packages(session): + if view.manifest_digest == manifest_digest: + return view + return None + + +def list_packages( + session: Session, release_id: Optional[str] = None +) -> Sequence[PackageView]: + """List packages, optionally for one release, oldest-first.""" + views = [ + view + for view in _iter_packages(session) + if release_id is None or view.release_id == release_id + ] + return sorted( + views, + key=lambda view: (view.created_at or datetime.min.replace(tzinfo=timezone.utc)), + ) + + +def row_to_manifest(view: PackageView) -> RuntimeManifestV2: + """Rehydrate a package view into its manifest.""" + return RuntimeManifestV2.model_validate(view.manifest_json) + + +def package_summary(view: PackageView) -> dict[str, Any]: + """Compact, non-secret package summary safe for responses.""" + manifest = view.manifest_json or {} + return { + "package_id": str(view.package_id), + "release_id": view.release_id, + "os": view.os, + "arch": view.arch, + "manifest_digest": view.manifest_digest, + "signature": view.signature, + "component_count": len(manifest.get("components") or []), + "supported_platforms": manifest.get("supported_platforms") or [], + "created_at": view.created_at.isoformat() if view.created_at else None, + } diff --git a/src/research/study/packaging/verifier.py b/src/research/study/packaging/verifier.py new file mode 100644 index 0000000..0dbfbc3 --- /dev/null +++ b/src/research/study/packaging/verifier.py @@ -0,0 +1,340 @@ +"""Package verification and path containment (Issue 11). + +``verify_package`` is default-deny: it enforces relative safe paths that stay +under the package root, SHA-256 and size integrity, executability expectations, +no undeclared executables, no secret files, an optional HMAC over the manifest, +and an optional self-check. Every failure is typed; a package that cannot be +fully verified is never considered usable. +""" + +from __future__ import annotations + +import hashlib +import hmac +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Optional + +from .enums import PackageReasonCode +from .models import ( + PackageIssue, + PackageVerificationResult, + RuntimeManifestV2, + normalize_sha256, +) + +__all__ = [ + "EXECUTABLE_SUFFIXES", + "SECRET_FILE_PATTERNS", + "PathContainment", + "hash_file", + "resolve_under_root", + "scan_secret_files", + "scan_undeclared_executables", + "verify_package", +] + +#: File suffixes treated as executables even without a POSIX execute bit. +EXECUTABLE_SUFFIXES = frozenset({".exe", ".bat", ".cmd", ".com", ".sh", ".ps1", ".command"}) + +#: Name components that must never appear in a participant package. +SECRET_FILE_PATTERNS = tuple( + re.compile(pattern) + for pattern in ( + r"(^|[._-])\.env(\.[a-z0-9]+)?$", + r"(^|[._-])id_(rsa|dsa|ecdsa|ed25519)($|[._-])", + r"(^|[._-])(credentials|credential|secret|secrets|token|tokens)($|[._-])", + r"(^|[._-])\.(npmrc|netrc|pypirc|git-credentials)$", + r"\.(pem|key|p12|pfx|jks|keystore)$", + ) +) + +#: High-confidence secret *content* patterns for small text files. +SECRET_CONTENT_PATTERNS = tuple( + re.compile(pattern) + for pattern in ( + r"-----BEGIN [A-Z ]*PRIVATE KEY-----", + r"\bsk-[A-Za-z0-9]{16,}", + r"\bghp_[A-Za-z0-9]{20,}", + r"\bxox[baprs]-[A-Za-z0-9-]{10,}", + r"AWS_SECRET_ACCESS_KEY\s*=", + ) +) + +_TEXT_SUFFIXES = frozenset( + {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf", ".txt", ".md", ".env", ".sh"} +) +_MAX_CONTENT_SCAN_BYTES = 256 * 1024 + +_SIGNATURE_PREFIXES = ("hmac-sha256:", "sha256:") + + +@dataclass +class PathContainment: + """Result of resolving a declared relative path under a trusted root.""" + + path: Optional[Path] = None + issue: Optional[PackageIssue] = None + errors: list[PackageIssue] = field(default_factory=list) + + @property + def ok(self) -> bool: + """Whether the path is contained and safe.""" + return self.path is not None and self.issue is None + + +def _issue(code: PackageReasonCode, message: str, field_name: str = "") -> PackageIssue: + return PackageIssue(code=code, message=message, field=field_name) + + +def resolve_under_root(root: Path, relative_path: str) -> PathContainment: + """Resolve ``relative_path`` under ``root``, rejecting escapes and symlink escapes.""" + if not relative_path or not relative_path.strip(): + return PathContainment(issue=_issue(PackageReasonCode.PATH_UNSAFE, "path must not be blank", relative_path)) + if "\x00" in relative_path: + return PathContainment(issue=_issue(PackageReasonCode.PATH_UNSAFE, "path must not contain NUL", relative_path)) + candidate = Path(relative_path) + if candidate.is_absolute(): + return PathContainment(issue=_issue(PackageReasonCode.PATH_ESCAPE, f"absolute path '{relative_path}' is not allowed", relative_path)) + if relative_path.startswith("~"): + return PathContainment(issue=_issue(PackageReasonCode.PATH_ESCAPE, f"home-relative path '{relative_path}' is not allowed", relative_path)) + if any(part == ".." for part in candidate.parts): + return PathContainment(issue=_issue(PackageReasonCode.PATH_ESCAPE, f"path '{relative_path}' contains '..'", relative_path)) + + resolved_root = root.resolve() + resolved = (resolved_root / candidate).resolve() + if resolved != resolved_root and resolved_root not in resolved.parents: + return PathContainment(issue=_issue(PackageReasonCode.PATH_ESCAPE, f"path '{relative_path}' escapes the package root", relative_path)) + if resolved.exists() and resolved.is_symlink(): + real = resolved.resolve() + if real != resolved_root and resolved_root not in real.parents: + return PathContainment(issue=_issue(PackageReasonCode.PATH_ESCAPE, f"path '{relative_path}' escapes the package root via a link", relative_path)) + return PathContainment(path=resolved) + + +def hash_file(path: Path) -> str: + """Stream a file and return its ``sha256:`` digest.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def _is_executable_file(path: Path) -> bool: + if not path.is_file(): + return False + if path.suffix.lower() in EXECUTABLE_SUFFIXES: + return True + return hasattr(os, "X_OK") and os.access(path, os.X_OK) + + +def scan_undeclared_executables(root: Path, declared: set[str]) -> list[str]: + """Return executable-looking files under ``root`` that no component declares.""" + declared_normalized = {_normalize_relative(value) for value in declared} + undeclared: list[str] = [] + for path in sorted(root.rglob("*")): + if not path.is_file() or not _is_executable_file(path): + continue + relative = _normalize_relative(str(path.relative_to(root))) + if relative not in declared_normalized: + undeclared.append(relative) + return undeclared + + +def scan_secret_files(root: Path) -> list[str]: + """Return files under ``root`` whose name or content looks like a secret.""" + findings: list[str] = [] + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + relative = str(path.relative_to(root)) + if any(part.startswith((".aws", ".ssh", ".gnupg")) for part in path.parts): + findings.append(relative) + continue + if any(pattern.search(path.name.lower()) for pattern in SECRET_FILE_PATTERNS): + findings.append(relative) + continue + if path.suffix.lower() in _TEXT_SUFFIXES and path.stat().st_size <= _MAX_CONTENT_SCAN_BYTES: + try: + content = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + if any(pattern.search(content) for pattern in SECRET_CONTENT_PATTERNS): + findings.append(relative) + return findings + + +def _normalize_relative(value: str) -> str: + return value.replace("\\", "/").lstrip("./") + + +def _signature_payload(manifest: RuntimeManifestV2) -> bytes: + from research.canonical import canonical_bytes + + return canonical_bytes(manifest.canonical_payload()) + + +def _verify_signature( + manifest: RuntimeManifestV2, secret: Optional[str] +) -> Optional[PackageIssue]: + if not manifest.signature: + return None # a signature is optional; when absent nothing is claimed + if not secret: + return _issue( + PackageReasonCode.SIGNATURE_MISMATCH, + "manifest claims a signature but no verification secret was supplied", + "signature", + ) + expected = hmac.new(secret.encode("utf-8"), _signature_payload(manifest), hashlib.sha256).hexdigest() + claimed = manifest.signature.strip().lower() + for prefix in _SIGNATURE_PREFIXES: + if claimed.startswith(prefix): + claimed = claimed[len(prefix) :] + break + if not hmac.compare_digest(claimed, expected): + return _issue( + PackageReasonCode.SIGNATURE_MISMATCH, + "manifest signature does not match the canonical payload", + "signature", + ) + return None + + +def sign_manifest(manifest: RuntimeManifestV2, secret: str) -> str: + """Return the HMAC signature over the manifest's canonical payload.""" + return "hmac-sha256:" + hmac.new( + secret.encode("utf-8"), _signature_payload(manifest), hashlib.sha256 + ).hexdigest() + + +def verify_package( + root: Path, + manifest: RuntimeManifestV2, + secret: Optional[str] = None, + now: Optional[object] = None, + *, + self_check_runner: Optional[Callable[[RuntimeManifestV2, Path], int]] = None, +) -> PackageVerificationResult: + """Verify a package directory against ``manifest``. Default-deny.""" + del now # validation is time-independent; kept for a stable call signature + errors: list[PackageIssue] = [] + + if manifest.schema_version != "2": + errors.append( + _issue( + PackageReasonCode.SCHEMA_VERSION_UNSUPPORTED, + f"manifest schema_version '{manifest.schema_version}' is not supported", + "schema_version", + ) + ) + if not manifest.digest_matches(): + errors.append( + _issue(PackageReasonCode.DIGEST_MISMATCH, "manifest content does not match manifest_digest", "manifest_digest") + ) + if not manifest.components: + errors.append(_issue(PackageReasonCode.EMPTY_COMPONENTS, "manifest declares no components", "components")) + + signature_issue = _verify_signature(manifest, secret) + if signature_issue is not None: + errors.append(signature_issue) + + if not root.is_dir(): + errors.append(_issue(PackageReasonCode.ARTIFACT_MISSING, f"package root '{root}' is not a directory", "root")) + return PackageVerificationResult(valid=False, manifest_digest=manifest.manifest_digest, errors=errors) + + verified = 0 + declared_paths: set[str] = set() + for component in manifest.components: + containment = resolve_under_root(root, component.relative_path) + if not containment.ok: + errors.append( + containment.issue + or _issue(PackageReasonCode.PATH_ESCAPE, "component path is not contained", component.name) + ) + continue + declared_paths.add(_normalize_relative(component.relative_path)) + path = containment.path + assert path is not None + if not path.is_file(): + errors.append( + _issue(PackageReasonCode.ARTIFACT_MISSING, f"component '{component.name}' file is missing", component.relative_path) + ) + continue + actual_size = path.stat().st_size + if actual_size != component.size: + errors.append( + _issue( + PackageReasonCode.SIZE_MISMATCH, + f"component '{component.name}' size {actual_size} != declared {component.size}", + component.relative_path, + ) + ) + continue + if normalize_sha256(hash_file(path)) != normalize_sha256(component.sha256): + errors.append( + _issue(PackageReasonCode.DIGEST_MISMATCH, f"component '{component.name}' digest mismatch", component.relative_path) + ) + continue + if component.executable and not _is_executable_file(path): + errors.append( + _issue(PackageReasonCode.NOT_EXECUTABLE, f"component '{component.name}' is declared executable but is not", component.relative_path) + ) + continue + verified += 1 + + for relative in scan_undeclared_executables(root, declared_paths): + errors.append( + _issue(PackageReasonCode.UNDECLARED_EXECUTABLE, f"executable '{relative}' is not declared by any component", relative) + ) + for relative in scan_secret_files(root): + errors.append( + _issue(PackageReasonCode.SECRET_FILE_PRESENT, f"secret-shaped file '{relative}' must not be packaged", relative) + ) + + if manifest.self_check is not None: + errors.extend(_verify_self_check(root, manifest, self_check_runner)) + + return PackageVerificationResult( + valid=not errors, + manifest_digest=manifest.manifest_digest, + verified_components=verified, + errors=errors, + ) + + +def _verify_self_check( + root: Path, + manifest: RuntimeManifestV2, + runner: Optional[Callable[[RuntimeManifestV2, Path], int]], +) -> list[PackageIssue]: + spec = manifest.self_check + assert spec is not None + if not spec.command: + return [_issue(PackageReasonCode.SELF_CHECK_FAILED, "self-check declares an empty command", "self_check.command")] + command_path = resolve_under_root(root, spec.command[0]) + if not command_path.ok or command_path.path is None or not command_path.path.exists(): + return [ + _issue( + PackageReasonCode.SELF_CHECK_FAILED, + f"self-check executable '{spec.command[0]}' is not present under the package root", + "self_check.command", + ) + ] + if runner is None: + return [] + try: + exit_code = runner(manifest, command_path.path) + except Exception as error: # noqa: BLE001 - a throwing self-check is a failure + return [_issue(PackageReasonCode.SELF_CHECK_FAILED, f"self-check raised: {error}", "self_check")] + if exit_code != spec.expected_exit_code: + return [ + _issue( + PackageReasonCode.SELF_CHECK_FAILED, + f"self-check exited {exit_code}, expected {spec.expected_exit_code}", + "self_check", + ) + ] + return [] diff --git a/src/research/study/protocol/__init__.py b/src/research/study/protocol/__init__.py new file mode 100644 index 0000000..4cebd3a --- /dev/null +++ b/src/research/study/protocol/__init__.py @@ -0,0 +1,89 @@ +"""Research study configuration validation and canonicalization. + +Public surface: + +* :mod:`research.study.protocol.enums` - closed vocabularies for configuration, + retention, telemetry and validation. +* :mod:`research.study.protocol.models` - Pydantic v2 contracts for + ``StudyProtocolV1`` and its versioned sub-objects, including the explicit + :class:`~research.study.protocol.models.ExplicitUnknown` state. +* :mod:`research.study.protocol.canonical` - deterministic, order-normalized JSON and + SHA-256 digest of a validated protocol (reusing the Issue 01 canonical + helpers). +* :mod:`research.study.protocol.validation` - typed field-level, cross-reference and + document-safety validation, plus the injected + :class:`~research.study.protocol.validation.ReleaseResolver` interface and its + :class:`~research.study.protocol.validation.NullReleaseResolver` default. +The persistence adapters live in :mod:`research.study.protocol.store` and take a +caller-supplied SQLAlchemy ``Session`` so this package stays free of any +application/App import. +""" + +from .canonical import protocol_canonical_json, protocol_digest +from .enums import ( + AssignmentStrategy, + AssignmentUnit, + CompletionPolicyKind, + ReleaseResolutionStatus, + RetentionAction, + ScheduleKind, + TelemetryFieldClass, + ValidationReasonCode, + ValidationSeverity, +) +from .models import ( + ExplicitUnknown, + FixedSchedule, + ResolvedDistribution, + RollingSchedule, + StudyProtocolV1, +) +from .validation import ( + DistributionResolution, + DistributionResolver, + NullDistributionResolver, + NullReleaseResolver, + ProtocolValidationError, + ReleaseResolution, + ReleaseResolver, + ValidationError, + blocking_errors, + freeze_protocol_distributions, + is_publishable, + normalized_condition_weights, + validate_protocol, + warnings, +) + +__all__ = [ + "AssignmentStrategy", + "AssignmentUnit", + "CompletionPolicyKind", + "DistributionResolution", + "DistributionResolver", + "ExplicitUnknown", + "FixedSchedule", + "NullDistributionResolver", + "NullReleaseResolver", + "ProtocolValidationError", + "ReleaseResolution", + "ReleaseResolutionStatus", + "ReleaseResolver", + "ResolvedDistribution", + "RetentionAction", + "RollingSchedule", + "ScheduleKind", + "StudyProtocolV1", + "TelemetryFieldClass", + "ValidationError", + "ValidationReasonCode", + "ValidationSeverity", + "blocking_errors", + "freeze_protocol_distributions", + "is_publishable", + "normalized_condition_weights", + "protocol_canonical_json", + "protocol_digest", + "validate_protocol", + "warnings", +] diff --git a/src/research/study/protocol/canonical.py b/src/research/study/protocol/canonical.py new file mode 100644 index 0000000..2943a5b --- /dev/null +++ b/src/research/study/protocol/canonical.py @@ -0,0 +1,159 @@ +"""Deterministic canonical serialization and SHA-256 digest for study protocols. + +This module reuses the Issue 01 canonical helpers +(:mod:`research.compatibility.canonical`) rather than reimplementing JSON +sorting and hashing. It adds one protocol-specific concern: normalizing +*set-like* arrays so that an equivalent document hashes identically no matter +how its lists were authored. + +Canonicalization rules +---------------------- + +* Map keys are sorted (delegated to ``canonical_json``). +* ``conditions`` are sorted by ``condition_id``; condition order is not + meaningful for execution because assignment selects by id and weight. +* ``environment_requirements.required_capabilities`` are sorted by + ``(capability, require_state)``, ``consent.locale_refs`` and + ``assignment.stratification`` are sorted lexically, and ``survey_hooks`` are + sorted by ``(survey_id, trigger)``. +* A condition's unset ``resolved_distribution`` is omitted: it is absent on a + draft and populated by the server at publication, so omitting it keeps a + draft's digest independent of an optional pin that is not part of the draft. +* Everything else is preserved as-is, including ``None`` (inherited/default) + and :class:`~research.study.protocol.models.ExplicitUnknown` values. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from research.compatibility.canonical import ( # re-exported for callers + canonical_bytes, + canonical_hash, + canonical_json, +) + +if TYPE_CHECKING: + from .models import StudyProtocolV1 + +__all__ = [ + "canonical_bytes", + "canonical_hash", + "canonical_json", + "canonicalize_protocol", + "protocol_canonical_bytes", + "protocol_canonical_json", + "protocol_digest", +] + + +def _sorted_list(values: Any, key) -> list[Any]: + if not isinstance(values, list): + return [] + return sorted(values, key=key) + + +def canonicalize_protocol(protocol: StudyProtocolV1) -> dict[str, Any]: + """Return the canonical, order-normalized mapping for ``protocol``.""" + data = protocol.model_dump(mode="json") + + conditions = data.get("conditions") + if isinstance(conditions, list): + data["conditions"] = sorted( + conditions, + key=lambda item: ( + str(item.get("condition_id", "")), + str(item.get("name") or ""), + ), + ) + # An unset frozen pin is semantically absent on a draft, so it is + # omitted from the canonical bytes rather than serialized as ``null``. + for item in data["conditions"]: + if not isinstance(item, dict): + continue + if item.get("resolved_distribution") is None: + item.pop("resolved_distribution", None) + + environment = data.get("environment_requirements") + if isinstance(environment, dict): + environment["required_capabilities"] = _sorted_list( + environment.get("required_capabilities"), + key=lambda item: ( + str(item.get("capability", "")), + str(item.get("require_state", "")), + ), + ) + + telemetry = data.get("telemetry_policy") + if isinstance(telemetry, dict): + telemetry["allowed_field_classes"] = _sorted_list( + telemetry.get("allowed_field_classes"), key=str + ) + + assignment = data.get("assignment") + if isinstance(assignment, dict) and isinstance( + assignment.get("stratification"), list + ): + assignment["stratification"] = _sorted_list( + assignment["stratification"], key=str + ) + + survey_hooks = data.get("survey_hooks") + if isinstance(survey_hooks, list): + data["survey_hooks"] = sorted( + survey_hooks, + key=lambda item: ( + str(item.get("survey_id", "")), + str(item.get("trigger", "")), + ), + ) + + governance = data.get("schema_governance") + if isinstance(governance, dict): + governance["compatible_from_schema_versions"] = _sorted_list( + governance.get("compatible_from_schema_versions"), key=str + ) + + return data + + +def _digest_mapping(protocol: StudyProtocolV1) -> dict[str, Any]: + """Canonical mapping with resolution metadata excluded from the digest. + + ``resolved_at`` records *when* a pin was resolved, not the pin's semantic + identity, so it is dropped from the digest preimage: re-publishing identical + content stays content-addressed regardless of wall-clock resolution time. + The timestamp is still persisted in the stored ``protocol_json``. + """ + data = canonicalize_protocol(protocol) + for item in data.get("conditions", []) or []: + if not isinstance(item, dict): + continue + frozen = item.get("resolved_distribution") + if isinstance(frozen, dict): + frozen.pop("resolved_at", None) + # An absent frozen profile config is semantically absent (legacy + # revisions), so it must not change a document's digest. + if frozen.get("agent_config") is None: + frozen.pop("agent_config", None) + return data + + +def protocol_canonical_json(protocol: StudyProtocolV1) -> str: + """Canonical JSON string for a protocol (sorted keys, compact separators).""" + return canonical_json(_digest_mapping(protocol)) + + +def protocol_canonical_bytes(protocol: StudyProtocolV1) -> bytes: + """UTF-8 encoded :func:`protocol_canonical_json`.""" + return protocol_canonical_json(protocol).encode("utf-8") + + +def protocol_digest(protocol: StudyProtocolV1) -> str: + """SHA-256 hex digest of the canonical protocol document.""" + return canonical_hash(_digest_mapping(protocol)) + + +def digest_matches(protocol: StudyProtocolV1, expected_digest: str) -> bool: + """Whether ``expected_digest`` equals the canonical digest of ``protocol``.""" + return bool(expected_digest) and protocol_digest(protocol) == expected_digest diff --git a/src/research/study/protocol/enums.py b/src/research/study/protocol/enums.py new file mode 100644 index 0000000..1d002ef --- /dev/null +++ b/src/research/study/protocol/enums.py @@ -0,0 +1,164 @@ +"""Closed vocabularies for validated study configuration. + +Every value here is part of a persisted configuration or API contract. +""" + +from __future__ import annotations + +from enum import Enum + + +class SchemaVersion(str, Enum): + """Supported ``StudyProtocolV1.schema_version`` values.""" + + V1 = "1" + + +class ScheduleKind(str, Enum): + """Whether a study runs to a fixed calendar window or a rolling duration.""" + + FIXED = "FIXED" + ROLLING = "ROLLING" + + +class AssignmentUnit(str, Enum): + """Unit that is randomized by the assignment service. + + Issue 02 only supports enrollment-level assignment. ``SESSION`` exists as an + additive contract value so a future protocol version can request it without + a breaking enum change; the V1 validator rejects it explicitly. + """ + + ENROLLMENT = "ENROLLMENT" + SESSION = "SESSION" + + +class AssignmentStrategy(str, Enum): + """How an assignment service picks a condition for one unit.""" + + WEIGHTED_RANDOM = "WEIGHTED_RANDOM" + # Salted-hash pick (``research.runtime.assignment.hashing``); every study + # created since the assignment-policy decision allocates with it. + DETERMINISTIC_HASH = "DETERMINISTIC_HASH" + STRATIFIED = "STRATIFIED" + # Equal-probability profile pick for one enrollment (studies created before + # the assignment-policy decision). + RANDOM_EQUAL = "RANDOM_EQUAL" + # Provenance label of an arm the study owner set by hand before first use; + # never allocated. + MANUAL = "MANUAL" + + +class RetentionAction(str, Enum): + """What happens to identifiable data when retention expires.""" + + RETAIN_ANONYMIZED = "RETAIN_ANONYMIZED" + DELETE_IDENTIFIABLE = "DELETE_IDENTIFIABLE" + DELETE_ALL = "DELETE_ALL" + + +class TelemetryFieldClass(str, Enum): + """Coarse class of telemetry fields the protocol permits capturing.""" + + STRUCTURAL = "STRUCTURAL" + CONTENT = "CONTENT" + METRICS = "METRICS" + DIAGNOSTICS = "DIAGNOSTICS" + + +class CompletionPolicyKind(str, Enum): + """Terminal condition that ends enrollment for a study.""" + + MANUAL = "MANUAL" + TARGET_CAPACITY = "TARGET_CAPACITY" + SCHEDULE_END = "SCHEDULE_END" + + +class ValidationSeverity(str, Enum): + """Whether a validation reason blocks publication, asks for review, or warns. + + ``ERROR`` blocks publication. ``NEEDS_REVIEW`` asks an operator to look + before proceeding and blocks an automatic publish. ``WARNING`` is + informational only: the document may publish, but the caller is expected to + surface the reason (for example an administrator selecting an unverified + distribution). + """ + + ERROR = "ERROR" + NEEDS_REVIEW = "NEEDS_REVIEW" + WARNING = "WARNING" + + +class ValidationReasonCode(str, Enum): + """Stable machine-readable reason a protocol cannot be published.""" + + # Schema / structural. + SCHEMA_INVALID = "SCHEMA_INVALID" + UNSUPPORTED_SCHEMA_VERSION = "UNSUPPORTED_SCHEMA_VERSION" + NO_CONDITIONS = "NO_CONDITIONS" + + # Conditions / weights. + DUPLICATE_CONDITION_ID = "DUPLICATE_CONDITION_ID" + NON_POSITIVE_WEIGHT = "NON_POSITIVE_WEIGHT" + EMPTY_WEIGHT_TOTAL = "EMPTY_WEIGHT_TOTAL" + + # Schedule. + FIXED_SCHEDULE_EXPIRED = "FIXED_SCHEDULE_EXPIRED" + FIXED_SCHEDULE_INVALID = "FIXED_SCHEDULE_INVALID" + ROLLING_DURATION_INVALID = "ROLLING_DURATION_INVALID" + + # Assignment. + ASSIGNMENT_UNIT_NOT_ENROLLMENT = "ASSIGNMENT_UNIT_NOT_ENROLLMENT" + ASSIGNMENT_STRATEGY_UNKNOWN = "ASSIGNMENT_STRATEGY_UNKNOWN" + ASSIGNMENT_STRATIFICATION_MISSING = "ASSIGNMENT_STRATIFICATION_MISSING" + + # Consent / privacy. + CONSENT_DIGEST_MISSING = "CONSENT_DIGEST_MISSING" + # Unsupported retention authoring: the published policy retains uploaded + # data according to consent; a participant-selected DELETE_ALL is not an + # authorable option. + RETENTION_UNSUPPORTED = "RETENTION_UNSUPPORTED" + + # Agent release references. + AGENT_RELEASE_UNPINNED = "AGENT_RELEASE_UNPINNED" + AGENT_RELEASE_LATEST = "AGENT_RELEASE_LATEST" + MUTABLE_AGENT_COMMAND = "MUTABLE_AGENT_COMMAND" + RELEASE_UNRESOLVED = "RELEASE_UNRESOLVED" + RELEASE_UNQUALIFIED = "RELEASE_UNQUALIFIED" + RELEASE_WITHDRAWN = "RELEASE_WITHDRAWN" + RELEASE_DIGEST_MISMATCH = "RELEASE_DIGEST_MISMATCH" + + # Agent profile references (a condition may pin an existing profile). + AGENT_PROFILE_NOT_FOUND = "AGENT_PROFILE_NOT_FOUND" + + # A distribution (AgentProfile) is not verified: its release has no + # passing producer tests or lacks a resolvable distribution identity. + DISTRIBUTION_UNVERIFIED = "DISTRIBUTION_UNVERIFIED" + + # Declared distribution contract on a release pin. + AGENT_RELEASE_MODE_UNKNOWN = "AGENT_RELEASE_MODE_UNKNOWN" + # A BYOA pin must carry a release identity (release_id/version), never a digest. + AGENT_RELEASE_IDENTITY_REQUIRED = "AGENT_RELEASE_IDENTITY_REQUIRED" + + # Environment requirements. + UNKNOWN_REQUIRED_CAPABILITY = "UNKNOWN_REQUIRED_CAPABILITY" + UNKNOWN_ENVIRONMENT_REQUIREMENT = "UNKNOWN_ENVIRONMENT_REQUIREMENT" + ENVIRONMENT_PROTOCOL_VERSION_MISSING = "ENVIRONMENT_PROTOCOL_VERSION_MISSING" + + # Telemetry / completion. + TELEMETRY_FIELD_CLASSES_EMPTY = "TELEMETRY_FIELD_CLASSES_EMPTY" + COMPLETION_TARGET_INVALID = "COMPLETION_TARGET_INVALID" + + # Document safety. + FORBIDDEN_IDENTIFIER = "FORBIDDEN_IDENTIFIER" + FORBIDDEN_CREDENTIAL = "FORBIDDEN_CREDENTIAL" + FORBIDDEN_LOCAL_PATH = "FORBIDDEN_LOCAL_PATH" + + +class ReleaseResolutionStatus(str, Enum): + """Outcome of resolving a pinned agent release through a resolver.""" + + RESOLVED = "RESOLVED" + NOT_FOUND = "NOT_FOUND" + UNQUALIFIED = "UNQUALIFIED" + WITHDRAWN = "WITHDRAWN" diff --git a/src/research/study/protocol/join_code.py b/src/research/study/protocol/join_code.py new file mode 100644 index 0000000..a6e1032 --- /dev/null +++ b/src/research/study/protocol/join_code.py @@ -0,0 +1,74 @@ +"""Human-shareable study join codes (researcher -> participant onboarding). + +A join code is a short, random, study-scoped handle that a researcher shares +with prospective participants. It is bound to exactly one *published* revision: +redeeming it enrolls the caller in that revision's study without ever letting +the client choose a revision, an eligibility verdict, or a consent document. + +Design constraints: + +* **Opaque and random.** The code is drawn from a CSPRNG and carries no study, + revision or account semantics. It can therefore be printed, spoken or typed + without leaking anything beyond "this study exists". +* **Human-typable.** The alphabet is Crockford base32 (no ``I``/``L``/``O``/``U``) + so an 8-character code is unambiguous to read aloud and to type. Normalization + folds the common look-alikes back to their canonical character. +* **Unique per published revision.** The database enforces uniqueness; the + in-process allocator retries on the astronomically unlikely collision. +* **Published-only.** Drafts never get a code, so an unpublished protocol can + never be joined. +""" + +from __future__ import annotations + +import re +import secrets + +__all__ = [ + "JOIN_CODE_ALPHABET", + "JOIN_CODE_LENGTH", + "generate_join_code", + "is_well_formed_join_code", + "normalize_join_code", +] + +#: Crockford base32: 0-9 and A-Z minus I, L, O and U. +JOIN_CODE_ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" + +#: Default code length. 32**8 == 2**40 possible codes. +JOIN_CODE_LENGTH = 8 + +_NON_CODE_CHARS = re.compile(r"[^0-9A-Z]") + +# Common look-alikes a human may type instead of the canonical character. +_LOOKALIKES = str.maketrans({"I": "1", "L": "1", "O": "0", "U": "V"}) + + +def generate_join_code(length: int = JOIN_CODE_LENGTH) -> str: + """Return a fresh random join code. + + The code is drawn from :data:`JOIN_CODE_ALPHABET` using a CSPRNG. It is not + derived from a study, revision or account identifier. + """ + if length <= 0: + raise ValueError("join code length must be positive") + return "".join(secrets.choice(JOIN_CODE_ALPHABET) for _ in range(length)) + + +def normalize_join_code(code: str) -> str: + """Return the canonical form of a user-supplied join code. + + Whitespace, separators and lower case are folded away, and Crockford + look-alikes (``I``/``L`` -> ``1``, ``O`` -> ``0``, ``U`` -> ``V``) are mapped + to their canonical character. An empty/garbage input normalizes to the empty + string, which never resolves. + """ + if not code: + return "" + upper = code.strip().upper().translate(_LOOKALIKES) + return _NON_CODE_CHARS.sub("", upper) + + +def is_well_formed_join_code(code: str, length: int = JOIN_CODE_LENGTH) -> bool: + """Whether ``code`` (already normalized) is exactly a join code.""" + return len(code) == length and all(char in JOIN_CODE_ALPHABET for char in code) diff --git a/src/research/study/protocol/models.py b/src/research/study/protocol/models.py new file mode 100644 index 0000000..3ce0f8a --- /dev/null +++ b/src/research/study/protocol/models.py @@ -0,0 +1,315 @@ +"""Pydantic v2 contracts for the versioned study protocol (``StudyProtocolV1``). + +Semantics that matter for reproducibility: + +* ``None`` means an *intentional inherited/default* value. It is never coerced + to a concrete value, and a missing field dumps back as ``None``. +* :class:`ExplicitUnknown` is the typed *insufficient evidence* state. It is a + distinct value from ``None`` and is never normalized to a default. +* Condition order is not meaningful for execution, so the canonical serializer + sorts conditions (and other set-like collections) by stable identity before + hashing. See :mod:`research.study.protocol.canonical`. + +The document is draft-only metadata plus policy. It must never carry a +participant/session/account identifier or a credential; those are rejected by +:mod:`research.study.protocol.validation`. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Any, Literal, Optional, Union +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import ( + AssignmentStrategy, + AssignmentUnit, + CompletionPolicyKind, + RetentionAction, + ScheduleKind, + TelemetryFieldClass, +) + +# Document-wide policy: unknown keys are rejected so a typo cannot silently +# drop a policy. Draft content is validated before it is ever persisted. +_BASE_CONFIG = ConfigDict(extra="forbid") + + +class ExplicitUnknown(BaseModel): + """Typed "insufficient evidence" marker. + + Using a model (rather than ``None`` or the string ``"unknown"``) means an + unknown value survives canonical serialization and round-trips as a + distinct state, so a consumer can never mistake it for an inherited default. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: Literal["UNKNOWN"] = "UNKNOWN" + + +class StudyMetadata(BaseModel): + """Draft-only, non-secret descriptive metadata.""" + + model_config = _BASE_CONFIG + + name: str + description: Optional[str] = None + owner: Optional[str] = None + + +class FixedSchedule(BaseModel): + """A study that accepts data between two absolute instants.""" + + model_config = _BASE_CONFIG + + kind: Literal[ScheduleKind.FIXED] = ScheduleKind.FIXED + start_at: datetime + end_at: Optional[datetime] = None + + +class RollingSchedule(BaseModel): + """A study whose window is a duration measured from enrollment.""" + + model_config = _BASE_CONFIG + + kind: Literal[ScheduleKind.ROLLING] = ScheduleKind.ROLLING + # Required and strictly positive for publication; ``None`` is an explicit + # missing value rather than an implicit default. + duration_seconds: Optional[int] = None + + +Schedule = Union[FixedSchedule, RollingSchedule] + + +class EnrollmentPolicy(BaseModel): + """How many units the study accepts and whether re-entry is allowed.""" + + model_config = _BASE_CONFIG + + # ``None`` = inherit the server default capacity; ExplicitUnknown = capacity + # is not yet known and must be resolved before publication. + capacity: Optional[Union[int, ExplicitUnknown]] = None + allow_reentry: Optional[bool] = None + + +class AssignmentPolicy(BaseModel): + """Legacy authoring policy retained only for protocol validation compatibility. + + Active runtime assignment uses immutable study profile selections and the + study's frozen ``assignment`` policy (``DETERMINISTIC_HASH``, or + ``RANDOM_EQUAL`` for older studies) in ``research.runtime.assignment``. + """ + + model_config = _BASE_CONFIG + + unit: AssignmentUnit = AssignmentUnit.ENROLLMENT + # Free string, not the enum: an unknown strategy must be preserved so the + # validator can reject it with a typed reason instead of failing to parse. + strategy: str = AssignmentStrategy.WEIGHTED_RANDOM.value + # Required when ``strategy`` is STRATIFIED; stable stratum key names. + stratification: Optional[list[str]] = None + + +class ResolvedAgentConfig(BaseModel): + """Legacy non-secret profile projection used by validation compatibility code. + + Active studies store equivalent configuration in ``study_agent_profile`` and + ``study_assignment`` snapshots; no publication or revision owns this data. + """ + + model_config = _BASE_CONFIG + + profile_id: UUID + name: str + model: str + framework_version: str + tools_json: str + approval_policy: str + max_steps: int + temperature: Optional[float] = None + max_context_tokens: Optional[int] = None + connection_id: Optional[UUID] = None + connection_label: Optional[str] = None + # The researcher who funds the connection grant for this condition: the + # study owner (``Study.created_by``) at publication. Non-secret and never an + # email/name, so it is safe to freeze and to include in the digest. + funding_owner_user_id: Optional[UUID] = None + + +class ResolvedDistribution(BaseModel): + """The distribution pin, frozen and written at publication. + + A draft condition only names an ``AgentProfile`` (a *distribution*) by its + opaque ``distribution_id``. Because distributions are mutable, admin-editable + rows, a published revision freezes the exact resolved pin here so an + immutable revision can always answer "which artifact produced this data?". + + ``PACKAGED`` distributions resolve to a digest-pinned registry release; a + participant-installed ``BYOA_EXTERNAL`` distribution carries the command / + package identity and no digest. ``verified`` is derived at resolution time + and is informational once frozen (the frozen value is never re-derived). + """ + + model_config = _BASE_CONFIG + + distribution_id: UUID + distribution_mode: str = "PACKAGED" + release_id: Optional[str] = None + agent_id: Optional[str] = None + version: Optional[str] = None + agent_package: Optional[str] = None + agent_command: Optional[str] = None + agent_command_args: list[str] = Field(default_factory=list) + artifact_digest: Optional[str] = None + verified: bool = False + resolved_at: Optional[datetime] = None + # Full, non-secret profile config frozen at publication. + agent_config: Optional[ResolvedAgentConfig] = None + + +class StudyCondition(BaseModel): + """One experimental arm: its weight and the distribution it runs. + + ``distribution_id`` names a server-side ``AgentProfile`` by opaque id only. + The profile is the *distribution*: it holds the provider ``base_url``, the + ``api_key_ref`` *name* (never the secret), the model and the artifact pin + (``release_id`` for ``PACKAGED``; ``agent_package``/``agent_command`` for + ``BYOA_EXTERNAL``). None of that is duplicated into the draft document. + + ``resolved_distribution`` is absent on a draft and is populated at + publication with the frozen, resolved pin (never authored by a caller). + """ + + model_config = _BASE_CONFIG + + condition_id: str + name: Optional[str] = None + # Relative, strictly positive frequency. Weights are normalized + # deterministically at assignment time (see + # :func:`research.study.protocol.validation.normalized_condition_weights`). + weight: float + # Opaque reference to an existing AgentProfile/"distribution" (validated by + # the backend, which is the only layer with database access). Never a secret. + distribution_id: UUID + adapter_version: Optional[str] = None + # Condition-scoped, non-secret overrides. Null/absent means "inherit". + declared_overrides: dict[str, Any] = Field(default_factory=dict) + # Frozen at publish; never supplied by a caller and rejected if present on a + # draft that has not been resolved by the server. + resolved_distribution: Optional[ResolvedDistribution] = None + + +class SessionPolicy(BaseModel): + """Timeouts that govern an enrolled session's lifecycle.""" + + model_config = _BASE_CONFIG + + idle_timeout_seconds: Optional[int] = None + resume_grace_seconds: Optional[int] = None + heartbeat_seconds: Optional[int] = None + + +class TelemetryPolicy(BaseModel): + """Which classes of telemetry fields the study permits capturing.""" + + model_config = _BASE_CONFIG + + allowed_field_classes: list[TelemetryFieldClass] = Field(default_factory=list) + + +class PrivacyPolicy(BaseModel): + """Retention and deletion behavior for enrolled data.""" + + model_config = _BASE_CONFIG + + retention_action: RetentionAction + retention_days: Optional[int] = None + + +class CapabilityExpectation(BaseModel): + """One required host/agent capability predicate. + + ``capability`` is a free string rather than an enum so an unknown-but- + required capability is preserved and flagged ``NEEDS_REVIEW`` instead of + being silently dropped by parsing. + """ + + model_config = _BASE_CONFIG + + capability: str + require_state: str = "SUPPORTED" + evidence_ref: Optional[str] = None + + +class EnvironmentRequirements(BaseModel): + """Host/agent compatibility a revision must satisfy before launch.""" + + model_config = _BASE_CONFIG + + # Expected negotiated ACP protocol version. Required for publication. + expected_protocol_version: Optional[str] = None + required_capabilities: list[CapabilityExpectation] = Field(default_factory=list) + # Untested-but-expected environment facts; explicit unknown is allowed and + # blocks publication with NEEDS_REVIEW. + host_kind: Optional[Union[str, ExplicitUnknown]] = None + + +class CompletionPolicy(BaseModel): + """When the study stops accepting new enrollments.""" + + model_config = _BASE_CONFIG + + policy: CompletionPolicyKind = CompletionPolicyKind.MANUAL + target_enrollments: Optional[int] = None + + +class SchemaGovernance(BaseModel): + """How the protocol schema itself may change for this study.""" + + model_config = _BASE_CONFIG + + owner: Optional[str] = None + change_policy: Optional[str] = None + compatible_from_schema_versions: list[str] = Field(default_factory=list) + + +class SurveyHook(BaseModel): + """Optional, versioned survey attachment point.""" + + model_config = _BASE_CONFIG + + survey_id: str + trigger: str + version: Optional[str] = None + + +class StudyProtocolV1(BaseModel): + """The complete, validated experimental contract (schema version ``1``).""" + + model_config = _BASE_CONFIG + + schema_version: str = "1" + study_id: UUID + metadata: StudyMetadata + # Explicit discriminated union: a document must choose fixed vs rolling. + schedule: Schedule = Field(discriminator="kind") + enrollment: EnrollmentPolicy = Field(default_factory=EnrollmentPolicy) + assignment: AssignmentPolicy = Field(default_factory=AssignmentPolicy) + conditions: list[StudyCondition] = Field(default_factory=list) + session_policy: SessionPolicy = Field(default_factory=SessionPolicy) + telemetry_policy: TelemetryPolicy = Field(default_factory=TelemetryPolicy) + privacy_policy: PrivacyPolicy + # Deprecated, tolerated only so existing protocol fixtures and the website + # editor payload keep parsing: consent is now a single global text accepted + # once at join and is never stored, versioned or checked per study. Remove + # this field once the editor and fixtures stop sending `consent`. + consent: Optional[dict[str, Any]] = None + environment_requirements: EnvironmentRequirements = Field( + default_factory=EnvironmentRequirements + ) + completion: CompletionPolicy = Field(default_factory=CompletionPolicy) + schema_governance: Optional[SchemaGovernance] = None + survey_hooks: list[SurveyHook] = Field(default_factory=list) diff --git a/src/research/study/protocol/store.py b/src/research/study/protocol/store.py new file mode 100644 index 0000000..9fcc68c --- /dev/null +++ b/src/research/study/protocol/store.py @@ -0,0 +1,388 @@ +"""CRUD-style persistence helpers for lifecycle-owned research studies. + +These functions take a caller-managed SQLAlchemy ``Session`` so the core package +never imports ``App`` or touches the application singleton. The router is +responsible for session lifecycle (``App.get_db_session`` / ``rollback`` / +``close``). +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional, Sequence + +from sqlalchemy import func, select + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + +from agents.tools import set_harness_profile_fields +from database.db_schemas import AgentProfile, Study as StudyRow +from database import crud as database_crud +from database.research_schemas import ( + ResearchEnrollment, + ResearchSessionV1, + StudyAgentProfile, + StudyAssignment, +) +from research.study.agents.distributions import validate_profile_configuration +from research.study.agents.enums import QualificationStatus +from research.study.agents.store import get_release, row_to_release +from research.canonical import canonical_hash + + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +@dataclass(frozen=True) +class StudyView: + """Study identity, backed by the real ``public.study`` row.""" + + study_id: uuid.UUID + name: str + description: Optional[str] + owner: Optional[str] + created_by: Optional[uuid.UUID] = None + is_research: bool = False + is_active: bool = False + starts_at: Optional[datetime] = None + ends_at: Optional[datetime] = None + research_status: Optional[str] = None + research_config_digest: Optional[str] = None + join_code: Optional[str] = None + consent_locked_at: Optional[datetime] = None + stopped_at: Optional[datetime] = None + stopped_by: Optional[str] = None + created_at: Optional[datetime] = None + profile_selections: list[dict[str, Any]] = field(default_factory=list) + + +@dataclass(frozen=True) +class StudyReadMetrics: + """Safe researcher-facing counts derived from persisted study rows.""" + + enrollment_count: int + active_enrollment_count: int + assignment_count: int + active_assignment_count: int + active_session_count: int + + +def _study_row_view( + row: StudyRow, profile_selections: Optional[list[dict[str, Any]]] = None +) -> StudyView: + return StudyView( + study_id=getattr(row, "study_id"), + name=getattr(row, "name"), + description=getattr(row, "description", None), + owner=None, + created_by=getattr(row, "created_by", None), + is_research=bool(getattr(row, "is_research", False)), + is_active=bool(getattr(row, "is_active", False)), + starts_at=getattr(row, "starts_at", None), + ends_at=getattr(row, "ends_at", None), + research_status=getattr(row, "research_status", None), + research_config_digest=getattr(row, "research_config_digest", None), + join_code=getattr(row, "join_code", None), + consent_locked_at=getattr(row, "consent_locked_at", None), + stopped_at=getattr(row, "stopped_at", None), + stopped_by=getattr(row, "stopped_by", None), + created_at=getattr(row, "created_at", None), + profile_selections=list(profile_selections or []), + ) + + +def _study_view(session: Session, row: StudyRow) -> StudyView: + selections = session.execute( + select(StudyAgentProfile) + .where(StudyAgentProfile.study_id == row.study_id) + .order_by(StudyAgentProfile.selection_order.asc()) + ).scalars().all() + return _study_row_view( + row, + [ + { + "profile_id": str(selection.profile_id), + "name": (selection.profile_snapshot_json or {}).get("name", ""), + "model": (selection.profile_snapshot_json or {}).get("model", ""), + "profile_digest": selection.profile_digest, + "selection_order": selection.selection_order, + } + for selection in selections + ], + ) + + +def build_profile_selections( + session: Session, + *, + study_id: uuid.UUID, + profile_ids: Optional[Sequence[uuid.UUID]], + created_by: Optional[uuid.UUID], + allow_shared_profiles: bool = False, + timestamp: Optional[datetime] = None, +) -> list[StudyAgentProfile]: + """Validate and build the frozen ``StudyAgentProfile`` rows for a study. + + The single implementation of the profile-freeze invariants (owner, active, + release-qualified, executable configuration) shared by :func:`create_study` + and the stopped-study clone. The returned rows are *not* added to the + session: the caller inserts them in its own transaction so study creation + and clone stay atomic. + """ + selected_profile_ids = list(profile_ids or []) + if len(selected_profile_ids) != len(set(selected_profile_ids)): + raise ValueError("study profile selection contains duplicates") + created_at = timestamp or _now() + selections: list[StudyAgentProfile] = [] + for selection_order, profile_id in enumerate(selected_profile_ids): + profile = session.get(AgentProfile, profile_id) + if profile is None or not bool(getattr(profile, "is_active", True)): + raise ValueError("selected agent profile is unavailable") + if not allow_shared_profiles and profile.owner_user_id != created_by: + raise PermissionError("selected agent profile is not owned by the researcher") + database_crud.validate_profile_release(session, profile.release_id) + if not profile.release_id: + raise ValueError("RELEASE_UNRESOLVED: selected profile has no release") + release = get_release(session, profile.release_id) + if release is None: + raise ValueError("RELEASE_UNRESOLVED: selected profile release is missing") + release_status = str(release.status or "").upper() + if release_status in { + QualificationStatus.RETIRED.value, + QualificationStatus.BLOCKED.value, + }: + raise ValueError("RELEASE_WITHDRAWN: selected profile release is withdrawn") + if release_status != QualificationStatus.QUALIFIED.value: + raise ValueError("RELEASE_NOT_QUALIFIED: selected profile release is not qualified") + # Framework, distribution mode/identity, tools and approval evidence must + # be executable *together* before the profile is frozen into the study + # (ISSUE-03/17). A qualified BYOA release stays a valid choice. + validate_profile_configuration( + profile, row_to_release(release), release_json=release.release_json + ) + snapshot = { + "profile_id": str(profile.profile_id), + "name": profile.name, + "model": profile.model, + "framework_version": profile.framework_version, + "release_id": profile.release_id, + "connection_id": str(profile.connection_id) if profile.connection_id else None, + "tools_json": profile.tools_json, + "approval_policy": profile.approval_policy, + "max_steps": profile.max_steps, + "temperature": profile.temperature, + "max_context_tokens": profile.max_context_tokens, + } + # Only a set prompt is frozen, so a profile without one keeps the exact + # snapshot (and profile_digest) it produced before the column existed. + if getattr(profile, "system_prompt", None) is not None: + snapshot["system_prompt"] = profile.system_prompt + # Likewise the built-in runtime's command/harness settings (D-01), + # validated above with the profile↔release contract. + snapshot.update(set_harness_profile_fields(profile)) + selections.append( + StudyAgentProfile( + study_id=study_id, + profile_id=profile.profile_id, + profile_digest=canonical_hash(snapshot), + profile_snapshot_json=snapshot, + selection_order=selection_order, + created_at=created_at, + ) + ) + return selections + + +def create_study( + session: Session, + *, + study_id: uuid.UUID, + name: str, + description: Optional[str] = None, + owner: Optional[str] = None, + created_by: Optional[uuid.UUID] = None, + starts_at: Optional[datetime] = None, + ends_at: Optional[datetime] = None, + is_research: bool = True, + default_config_id: Optional[int] = None, + research_status: Optional[str] = None, + research_config_json: Optional[dict[str, Any]] = None, + research_config_digest: Optional[str] = None, + join_code: Optional[str] = None, + profile_ids: Optional[Sequence[uuid.UUID]] = None, + allow_shared_profiles: bool = False, + now: Optional[datetime] = None, + inference_budget_default_micro_usd: int = 0, + inference_budget_warning_fraction: Optional[Any] = None, + inference_budget_updated_by: Optional[str] = None, +) -> StudyView: + """Insert the real ``public.study`` identity row. + + Agent research studies set ``is_research`` and never fabricate a completion + ``default_config_id``. Research rows start in ``DRAFT`` unless a caller + explicitly supplies another lifecycle state. + """ + timestamp = now or _now() + selected_profile_ids = list(profile_ids or []) + config = dict(research_config_json or {}) + if selected_profile_ids: + config["profile_ids"] = [str(profile_id) for profile_id in selected_profile_ids] + config_digest = research_config_digest or canonical_hash(config) + row = StudyRow( + study_id=study_id, + name=name, + description=description, + created_by=created_by, + starts_at=starts_at or timestamp, + ends_at=ends_at, + is_active=False, + default_config_id=default_config_id, + is_research=is_research, + research_status=(research_status or ("DRAFT" if is_research else None)), + research_config_json=config, + research_config_digest=config_digest, + join_code=join_code, + created_at=timestamp, + # Participant budgets (shared provider key): editable later and kept + # outside the digested research configuration on purpose. + inference_budget_default_micro_usd=int(inference_budget_default_micro_usd or 0), + inference_budget_updated_at=timestamp if inference_budget_default_micro_usd else None, + inference_budget_updated_by=inference_budget_updated_by, + ) + if inference_budget_warning_fraction is not None: + row.inference_budget_warning_fraction = inference_budget_warning_fraction + session.add(row) + for selection in build_profile_selections( + session, + study_id=study_id, + profile_ids=selected_profile_ids, + created_by=created_by, + allow_shared_profiles=allow_shared_profiles, + timestamp=timestamp, + ): + session.add(selection) + session.commit() + session.refresh(row) + return _study_view(session, row) + + +def get_study(session: Session, study_id: uuid.UUID) -> Optional[StudyView]: + """Fetch a study identity row by id, or ``None``.""" + row = session.get(StudyRow, study_id) + return _study_view(session, row) if row is not None else None + + +def get_study_read_metrics(session: Session, study_id: uuid.UUID) -> StudyReadMetrics: + """Return persisted counts without selecting private participant fields.""" + enrollment_count, active_enrollment_count = session.execute( + select( + func.count(ResearchEnrollment.enrollment_id), + func.count(ResearchEnrollment.enrollment_id).filter( + ResearchEnrollment.status == "ACTIVE" + ), + ).where(ResearchEnrollment.study_id == study_id) + ).one() + assignment_count, active_assignment_count = session.execute( + select( + func.count(StudyAssignment.assignment_id), + func.count(StudyAssignment.assignment_id).filter( + StudyAssignment.status == "ACTIVE" + ), + ).where(StudyAssignment.study_id == study_id) + ).one() + active_session_count = session.execute( + select(func.count(ResearchSessionV1.session_id)).where( + ResearchSessionV1.study_id == study_id, + ResearchSessionV1.state.not_in(("ended", "revoked")), + ) + ).scalar_one() + return StudyReadMetrics( + enrollment_count=int(enrollment_count), + active_enrollment_count=int(active_enrollment_count), + assignment_count=int(assignment_count), + active_assignment_count=int(active_assignment_count), + active_session_count=int(active_session_count), + ) + + +def list_studies( + session: Session, owner_user_id: Optional[uuid.UUID] = None +) -> Sequence[StudyView]: + """List studies, owner-scoped when ``owner_user_id`` is given.""" + statement = select(StudyRow) + if owner_user_id is not None: + statement = statement.where(StudyRow.created_by == owner_user_id) + statement = statement.order_by(StudyRow.created_at.asc()) + return [_study_view(session, row) for row in session.execute(statement).scalars().all()] + + +def set_study_active( + session: Session, study_id: uuid.UUID, active: bool +) -> Optional[StudyRow]: + """Apply an ordinary activation/deactivation lifecycle transition. + + Activation is a plain projection update: it sets ``is_active`` alongside + ``research_status`` and imposes no owner-level study limit, so several + studies may be ACTIVE for the same owner at once. A stopped study is + terminal and cannot be reactivated (``ValueError``). + """ + row = session.get(StudyRow, study_id) + if row is None: + return None + if ( + active + and getattr(row, "research_status", None) == "STUDY_STOPPED" + ): + raise ValueError("stopped research studies cannot be reactivated") + setattr(row, "is_active", active) + if getattr(row, "is_research", False): + setattr(row, "research_status", "ACTIVE" if active else "DRAFT") + session.commit() + session.refresh(row) + return row + + +def research_study_is_open(study: Any, now: Optional[datetime] = None) -> bool: + """Whether a research study is currently open for execution. + + ``is_active`` alone is a writer-set flag; an ended window must refuse + behaviourally even before a writer flips the flag, so the reader path checks + the schedule too. + """ + if study is None or not getattr(study, "is_research", False): + return False + if not getattr(study, "is_active", False): + return False + timestamp = now or _now() + starts_at = getattr(study, "starts_at", None) + ends_at = getattr(study, "ends_at", None) + if starts_at is not None and timestamp < starts_at: + return False + if ends_at is not None and timestamp > ends_at: + return False + return True + + +def get_study_by_join_code( + session: Session, join_code: str +) -> Optional[Any]: + """Fetch a study by its study-owned join code.""" + normalized = str(join_code or "").strip().upper() + if not normalized: + return None + return session.execute( + select(StudyRow).where(StudyRow.join_code == normalized) + ).scalars().first() + + +def get_study_join_code( + session: Session, study_id: uuid.UUID +) -> Optional[Any]: + """Fetch the study-owned join code, or ``None``.""" + row = session.get(StudyRow, study_id) + return row if row is not None and getattr(row, "join_code", None) else None diff --git a/src/research/study/protocol/validation.py b/src/research/study/protocol/validation.py new file mode 100644 index 0000000..9560bf9 --- /dev/null +++ b/src/research/study/protocol/validation.py @@ -0,0 +1,988 @@ +"""Field-level, cross-reference and safety validation for study protocols. + +Validation never fails open. Every rejection is a typed :class:`ValidationError` +with a stable machine-readable +:class:`~research.study.protocol.enums.ValidationReasonCode` and a dotted ``field`` +path. Cross-validation against the agent registry happens through an injected +:class:`ReleaseResolver`, so this package never imports the registry. + +Weight normalization +-------------------- + +Condition weights are *relative* frequencies, not required to sum to one. The +deterministic normalization is ``w_i / sum(w)`` and is exposed through +:func:`normalized_condition_weights`. A protocol whose weights do not form a +normalizable positive total (missing, non-positive per condition, or a +non-positive/NaN total) is rejected with ``NON_POSITIVE_WEIGHT`` / +``EMPTY_WEIGHT_TOTAL`` rather than silently repaired. +""" + +from __future__ import annotations + +import math +import re +from datetime import datetime, timezone +from typing import Any, Mapping, Optional, Protocol, Union +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field +from pydantic import ValidationError as PydanticValidationError + +from research.compatibility.enums import CapabilityId, CapabilityState + +from .enums import ( + AssignmentStrategy, + AssignmentUnit, + CompletionPolicyKind, + ReleaseResolutionStatus, + RetentionAction, + ValidationReasonCode, + ValidationSeverity, +) +from .models import ( + ExplicitUnknown, + FixedSchedule, + ResolvedAgentConfig, + ResolvedDistribution, + RollingSchedule, + StudyCondition, + StudyProtocolV1, +) + +_VALID_SCHEMA_VERSIONS = frozenset({"1"}) + +#: Known distribution contracts a release pin may declare. +_DISTRIBUTION_MODES = frozenset({"PACKAGED", "BYOA_EXTERNAL"}) + +# Key substrings that identify participant/session/account data or credentials. +# ``_SAFE_KEY_NAMES`` lists legitimate, non-secret keys that merely *contain* a +# flagged substring (e.g. a token count), so they are not false positives. +_SAFE_KEY_NAMES = frozenset( + { + "max_context_tokens", + "max_tokens", + "prompt_tokens", + "completion_tokens", + "total_tokens", + # The funding owner is deliberately frozen (non-secret) so connection + # authorization runs against the study owner, not the participant. It is + # a UUID, never an email/name, and never a credential. + "funding_owner_user_id", + } +) +_IDENTIFIER_KEY_PARTS = ( + "participant", + "session_id", + "session_token", + "user_id", + "account_id", + "subject_id", +) +_CREDENTIAL_KEY_PARTS = ( + "credential", + "password", + "passwd", + "secret", + "token", + "api_key", + "apikey", + "authorization", + "private_key", + "access_key", + "client_secret", + "cookie", +) +_CREDENTIAL_VALUE_PATTERNS = ( + re.compile(r"\bsk-[A-Za-z0-9]{8,}"), + re.compile(r"\bghp_[A-Za-z0-9]{8,}"), + re.compile(r"\bAKIA[A-Z0-9]{8,}"), + re.compile(r"\bBearer\s+[A-Za-z0-9._-]{8,}", re.IGNORECASE), +) + +# Local absolute filesystem paths are never part of a portable, participant- +# independent protocol (Issue 02 section 8): they leak a machine layout and +# cannot be resolved on another host. Detection is intentionally conservative +# so URLs and relative references are not rejected. +_LOCAL_PATH_PATTERNS = ( + re.compile(r"^[A-Za-z]:[\\/]"), + re.compile( + r"^/(?:Users|home|root|Volumes|private|opt|usr|etc|var|tmp|Applications)(?:/|$)" + ), + re.compile(r"(?:^|[\s\"'(=])/(?:Users|home|root)/"), +) + +_KNOWN_CAPABILITIES = frozenset(capability.value for capability in CapabilityId) +_KNOWN_CAPABILITY_STATES = frozenset(state.value for state in CapabilityState) + + +class ProtocolValidationError(ValueError): + """Raised by imperative helpers; ``validate_protocol`` never raises for data.""" + + +class ValidationError(BaseModel): + """One typed reason a protocol cannot be published.""" + + model_config = ConfigDict(extra="forbid") + + code: ValidationReasonCode + field: str + message: str + severity: ValidationSeverity = ValidationSeverity.ERROR + + +class ReleaseResolution(BaseModel): + """Outcome of resolving one pinned agent release.""" + + model_config = ConfigDict(extra="forbid") + + status: ReleaseResolutionStatus + agent_id: str + release_id: Optional[str] = None + version: Optional[str] = None + artifact_digest: Optional[str] = None + distribution_mode: str = "PACKAGED" + message: Optional[str] = None + + +class ReleaseResolver(Protocol): + """Interface the registry (or a test double) implements for cross-checks.""" + + def resolve( + self, + agent_id: str, + *, + release_id: Optional[str] = None, + version: Optional[str] = None, + ) -> ReleaseResolution: # pragma: no cover - structural protocol + ... + + +class NullReleaseResolver: + """Resolver that performs no external lookup. + + A reference is treated as resolved when it is *qualified* (has a + ``release_id`` or ``version``). Digest pinning and ``latest`` are enforced + structurally by the validator, so this is the correct default when no + registry backend is wired up: it never invents a digest and never fails open + on an unqualified reference. + """ + + def resolve( + self, + agent_id: str, + *, + release_id: Optional[str] = None, + version: Optional[str] = None, + ) -> ReleaseResolution: + if not release_id and not version: + return ReleaseResolution( + status=ReleaseResolutionStatus.UNQUALIFIED, + agent_id=agent_id, + release_id=release_id, + version=version, + message="release reference names no release_id or version", + ) + return ReleaseResolution( + status=ReleaseResolutionStatus.RESOLVED, + agent_id=agent_id, + release_id=release_id, + version=version, + ) + + +class DistributionResolution(BaseModel): + """Resolved, non-secret view of one distribution (an ``AgentProfile``). + + Produced by a :class:`DistributionResolver` that has database access to the + profile row and the release registry. The protocol validator never touches + the database; it consumes this view to emit typed reasons. ``verified`` is + DERIVED by the resolver from conformance evidence (never caller-supplied), + and ``PACKAGED`` distributions additionally expose the resolved release + identity and (platform-selected) artifact digest. ``BYOA_EXTERNAL`` + distributions expose a command/package identity and are never verified. + """ + + model_config = ConfigDict(extra="forbid") + + found: bool = False + distribution_id: Optional[UUID] = None + distribution_mode: str = "PACKAGED" + release_id: Optional[str] = None + agent_id: Optional[str] = None + version: Optional[str] = None + artifact_digest: Optional[str] = None + agent_package: Optional[str] = None + agent_command: Optional[str] = None + agent_command_args: list[str] = Field(default_factory=list) + verified: bool = False + release_status: ReleaseResolutionStatus = ReleaseResolutionStatus.NOT_FOUND + message: Optional[str] = None + # Full non-secret profile config, frozen at publication (optional so pure + # protocol tests and legacy revisions remain valid). + agent_config: Optional[ResolvedAgentConfig] = None + + def to_frozen(self, *, resolved_at: Optional[datetime] = None) -> ResolvedDistribution: + """Return the immutable pin written into a published revision.""" + return ResolvedDistribution( + distribution_id=self.distribution_id + or UUID(int=0), + distribution_mode=self.distribution_mode, + release_id=self.release_id, + agent_id=self.agent_id, + version=self.version, + agent_package=self.agent_package, + agent_command=self.agent_command, + agent_command_args=list(self.agent_command_args), + artifact_digest=self.artifact_digest, + verified=self.verified, + resolved_at=resolved_at, + agent_config=self.agent_config, + ) + + +class DistributionResolver(Protocol): + """Interface a distribution backend (or a test double) implements.""" + + def resolve(self, distribution_id: UUID) -> DistributionResolution: # pragma: no cover + ... + + +class NullDistributionResolver: + """Resolver that performs no external lookup. + + Without a wired backend a *draft* cannot be resolved, so every distribution + is reported as not found. A published document that already carries its + frozen ``resolved_distribution`` is validated from that pin directly (see + :func:`_resolve_distribution`), so this default is correct for pure protocol + tests and never fails open on an unqualified draft. + """ + + def resolve(self, distribution_id: UUID) -> DistributionResolution: + return DistributionResolution(found=False, distribution_id=distribution_id) + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def _iter_nodes(value: Any, path: str = "") -> Any: + """Yield ``(path, key, scalar)`` triples for every node in ``value``.""" + if isinstance(value, Mapping): + for key, item in value.items(): + child = f"{path}.{key}" if path else str(key) + yield child, str(key), item + yield from _iter_nodes(item, child) + elif isinstance(value, list): + for index, item in enumerate(value): + child = f"{path}[{index}]" + yield from _iter_nodes(item, child) + + +def _scan_document(data: Any) -> list[ValidationError]: + """Reject participant/session/account identifiers and credentials.""" + errors: list[ValidationError] = [] + seen: set[tuple[str, str]] = set() + + for path, key, value in _iter_nodes(data): + lowered = key.lower() + if lowered in _SAFE_KEY_NAMES: + continue + if any(part in lowered for part in _IDENTIFIER_KEY_PARTS): + signature = (path, "identifier") + if signature not in seen: + seen.add(signature) + errors.append( + ValidationError( + code=ValidationReasonCode.FORBIDDEN_IDENTIFIER, + field=path, + message=( + "protocol must not contain participant/session/account " + f"identifiers (found key {key!r})" + ), + ) + ) + elif any(part in lowered for part in _CREDENTIAL_KEY_PARTS): + signature = (path, "credential") + if signature not in seen: + seen.add(signature) + errors.append( + ValidationError( + code=ValidationReasonCode.FORBIDDEN_CREDENTIAL, + field=path, + message=( + f"protocol must not contain credentials (found key {key!r})" + ), + ) + ) + elif isinstance(value, str) and any( + pattern.search(value) for pattern in _CREDENTIAL_VALUE_PATTERNS + ): + signature = (path, "credential-value") + if signature not in seen: + seen.add(signature) + errors.append( + ValidationError( + code=ValidationReasonCode.FORBIDDEN_CREDENTIAL, + field=path, + message="protocol value looks like a credential", + ) + ) + elif isinstance(value, str) and any( + pattern.search(value) for pattern in _LOCAL_PATH_PATTERNS + ): + signature = (path, "local-path") + if signature not in seen: + seen.add(signature) + errors.append( + ValidationError( + code=ValidationReasonCode.FORBIDDEN_LOCAL_PATH, + field=path, + message="protocol values must not contain local absolute paths", + ) + ) + + return errors + + +def _schema_error(error: PydanticValidationError) -> ValidationError: + details = error.errors() + first = details[0] if details else {} + location = first.get("loc") or () + field = ".".join(str(part) for part in location) + return ValidationError( + code=ValidationReasonCode.SCHEMA_INVALID, + field=field, + message=str(first.get("msg", "protocol document failed schema validation")), + ) + + +def _is_latest(value: Optional[str]) -> bool: + return bool(value) and str(value).strip().lower() == "latest" + + +def _check_conditions(protocol: StudyProtocolV1) -> list[ValidationError]: + errors: list[ValidationError] = [] + conditions = protocol.conditions + + if not conditions: + errors.append( + ValidationError( + code=ValidationReasonCode.NO_CONDITIONS, + field="conditions", + message="a study protocol must declare at least one condition", + ) + ) + return errors + + seen: set[str] = set() + for index, condition in enumerate(conditions): + field = f"conditions[{index}]" + if condition.condition_id in seen: + errors.append( + ValidationError( + code=ValidationReasonCode.DUPLICATE_CONDITION_ID, + field=f"{field}.condition_id", + message=f"condition_id {condition.condition_id!r} is duplicated", + ) + ) + seen.add(condition.condition_id) + + weight = condition.weight + if weight is None or not math.isfinite(weight) or weight <= 0: + errors.append( + ValidationError( + code=ValidationReasonCode.NON_POSITIVE_WEIGHT, + field=f"{field}.weight", + message="condition weight must be a finite positive number", + ) + ) + + finite_weights = [ + condition.weight + for condition in conditions + if condition.weight is not None and math.isfinite(condition.weight) + ] + if not finite_weights or sum(finite_weights) <= 0: + errors.append( + ValidationError( + code=ValidationReasonCode.EMPTY_WEIGHT_TOTAL, + field="conditions", + message="condition weights must sum to a positive total", + ) + ) + + return errors + + +def _check_schedule(protocol: StudyProtocolV1, now: datetime) -> list[ValidationError]: + schedule = protocol.schedule + if isinstance(schedule, FixedSchedule): + if schedule.end_at is not None and schedule.end_at <= schedule.start_at: + return [ + ValidationError( + code=ValidationReasonCode.FIXED_SCHEDULE_INVALID, + field="schedule.end_at", + message="fixed schedule end_at must be after start_at", + ) + ] + if schedule.end_at is not None and schedule.end_at < now: + return [ + ValidationError( + code=ValidationReasonCode.FIXED_SCHEDULE_EXPIRED, + field="schedule.end_at", + message="fixed schedule already ended", + ) + ] + return [] + + if isinstance(schedule, RollingSchedule): + if schedule.duration_seconds is None or schedule.duration_seconds <= 0: + return [ + ValidationError( + code=ValidationReasonCode.ROLLING_DURATION_INVALID, + field="schedule.duration_seconds", + message="rolling schedule duration_seconds must be present and > 0", + ) + ] + return [] + + return [ + ValidationError( + code=ValidationReasonCode.SCHEMA_INVALID, + field="schedule.kind", + message="schedule must be FIXED or ROLLING", + ) + ] + + +def _check_assignment(protocol: StudyProtocolV1) -> list[ValidationError]: + errors: list[ValidationError] = [] + assignment = protocol.assignment + + if assignment.unit != AssignmentUnit.ENROLLMENT: + errors.append( + ValidationError( + code=ValidationReasonCode.ASSIGNMENT_UNIT_NOT_ENROLLMENT, + field="assignment.unit", + message="issue 02 only supports enrollment-level assignment", + ) + ) + + known_strategies = {strategy.value for strategy in AssignmentStrategy} + if assignment.strategy not in known_strategies: + errors.append( + ValidationError( + code=ValidationReasonCode.ASSIGNMENT_STRATEGY_UNKNOWN, + field="assignment.strategy", + message=f"unknown assignment strategy {assignment.strategy!r}", + ) + ) + elif ( + assignment.strategy == AssignmentStrategy.STRATIFIED.value + and not assignment.stratification + ): + errors.append( + ValidationError( + code=ValidationReasonCode.ASSIGNMENT_STRATIFICATION_MISSING, + field="assignment.stratification", + message="stratified assignment requires stratification keys", + ) + ) + + return errors + + +def _resolve_distribution( + condition: StudyCondition, + resolver: Optional[DistributionResolver], +) -> DistributionResolution: + """Resolve one condition's distribution to its non-secret view. + + A wired :class:`DistributionResolver` is authoritative. Without one, a + published revision's frozen ``resolved_distribution`` is trusted + structurally; a draft that only names an id is reported as not found. + """ + if resolver is not None: + return resolver.resolve(condition.distribution_id) + frozen = condition.resolved_distribution + if frozen is None: + return DistributionResolution( + found=False, distribution_id=condition.distribution_id + ) + return DistributionResolution( + found=True, + distribution_id=frozen.distribution_id, + distribution_mode=frozen.distribution_mode, + release_id=frozen.release_id, + agent_id=frozen.agent_id, + version=frozen.version, + artifact_digest=frozen.artifact_digest, + agent_package=frozen.agent_package, + agent_command=frozen.agent_command, + agent_command_args=list(frozen.agent_command_args), + verified=frozen.verified, + release_status=( + ReleaseResolutionStatus.RESOLVED + if frozen.verified + else ReleaseResolutionStatus.UNQUALIFIED + ), + ) + + +def _unverified_error( + field: str, + *, + distribution: DistributionResolution, + actor_is_admin: bool, +) -> ValidationError: + """One typed reason a distribution is not verified. + + A non-administrator researcher may only author a condition from a VERIFIED + distribution, so this is a hard ``ERROR`` for them. An administrator may + deliberately select an unverified distribution: it is flagged as a + ``WARNING`` that publish surfaces but does not block. + """ + mode = str(distribution.distribution_mode or "PACKAGED").strip().upper() + if mode == "BYOA_EXTERNAL": + message = ( + "the BYOA release lacks passing tests or a supported agent identity" + ) + else: + message = ( + "the distribution's release has no passing producer tests (unverified)" + ) + return ValidationError( + code=ValidationReasonCode.DISTRIBUTION_UNVERIFIED, + field=field, + message=message, + severity=( + ValidationSeverity.WARNING + if actor_is_admin + else ValidationSeverity.ERROR + ), + ) + + +def _check_distributions( + protocol: StudyProtocolV1, + resolver: Optional[DistributionResolver], + actor_is_admin: bool, +) -> list[ValidationError]: + errors: list[ValidationError] = [] + + for index, condition in enumerate(protocol.conditions): + field = f"conditions[{index}].distribution_id" + frozen = condition.resolved_distribution + + distribution = _resolve_distribution(condition, resolver) + if not distribution.found: + errors.append( + ValidationError( + code=ValidationReasonCode.AGENT_PROFILE_NOT_FOUND, + field=field, + message=( + f"distribution {condition.distribution_id} does not exist" + ), + ) + ) + continue + + mode = str(distribution.distribution_mode or "PACKAGED").strip().upper() + if mode not in _DISTRIBUTION_MODES: + errors.append( + ValidationError( + code=ValidationReasonCode.AGENT_RELEASE_MODE_UNKNOWN, + field=f"{field}.distribution_mode", + message=( + "distribution_mode must be one of " + + ", ".join(sorted(_DISTRIBUTION_MODES)) + ), + ) + ) + continue + + if _is_latest(distribution.release_id) or _is_latest(distribution.version): + errors.append( + ValidationError( + code=ValidationReasonCode.AGENT_RELEASE_LATEST, + field=field, + message="a distribution must pin a release, not 'latest'", + ) + ) + + if mode == "BYOA_EXTERNAL": + # A participant-installed agent is resolved from a command/package + # identity, never from an artifact digest. + if not ( + distribution.release_id + or distribution.agent_package + or distribution.agent_command + or distribution.agent_id + ): + errors.append( + ValidationError( + code=ValidationReasonCode.AGENT_RELEASE_IDENTITY_REQUIRED, + field=field, + message=( + "a BYOA_EXTERNAL distribution must declare an " + "agent_package or agent_command identity" + ), + ) + ) + # No conformance evidence can be bound to a participant-installed + # agent, so a BYOA distribution is always unverified. + errors.append( + _unverified_error( + field, distribution=distribution, actor_is_admin=actor_is_admin + ) + ) + continue + + # PACKAGED: the distribution MUST pin a resolvable release id. + if not distribution.release_id: + errors.append( + ValidationError( + code=ValidationReasonCode.AGENT_RELEASE_UNPINNED, + field=field, + message="a PACKAGED distribution must pin a release_id", + ) + ) + continue + + # A PACKAGED pin must never embed a mutable executable command. + if frozen is not None and frozen.agent_command: + errors.append( + ValidationError( + code=ValidationReasonCode.MUTABLE_AGENT_COMMAND, + field=f"{field}.resolved_distribution.agent_command", + message=( + "a PACKAGED distribution must not embed an executable " + "command" + ), + ) + ) + + if distribution.release_status == ReleaseResolutionStatus.NOT_FOUND: + errors.append( + ValidationError( + code=ValidationReasonCode.RELEASE_UNRESOLVED, + field=field, + message=( + f"release {distribution.release_id!r} could not be resolved" + ), + ) + ) + elif distribution.release_status == ReleaseResolutionStatus.WITHDRAWN: + errors.append( + ValidationError( + code=ValidationReasonCode.RELEASE_WITHDRAWN, + field=field, + message=( + f"release {distribution.release_id!r} has been withdrawn" + ), + ) + ) + elif not distribution.verified: + # The release exists but lacks passing producer tests. + errors.append( + _unverified_error( + field, distribution=distribution, actor_is_admin=actor_is_admin + ) + ) + elif not distribution.artifact_digest: + # A verified PACKAGED distribution must still resolve to a concrete + # digest-pinned artifact; without one there is nothing to launch. + errors.append( + ValidationError( + code=ValidationReasonCode.AGENT_RELEASE_UNPINNED, + field=field, + message=( + f"release {distribution.release_id!r} publishes no " + "digest-pinned artifact" + ), + ) + ) + + # A frozen digest that disagrees with the registry is a hard block: the + # immutable revision no longer describes the artifact it claims to. + if ( + frozen is not None + and frozen.artifact_digest + and distribution.artifact_digest + and frozen.artifact_digest != distribution.artifact_digest + ): + errors.append( + ValidationError( + code=ValidationReasonCode.RELEASE_DIGEST_MISMATCH, + field=f"{field}.resolved_distribution.artifact_digest", + message="frozen artifact digest does not match the registry", + ) + ) + + return errors + + +def freeze_protocol_distributions( + protocol: StudyProtocolV1, + resolver: Optional[DistributionResolver], + *, + now: Optional[datetime] = None, +) -> StudyProtocolV1: + """Return a copy of ``protocol`` with every condition's pin frozen. + + The resolved pin (release identity, artifact digest, distribution mode and + the derived ``verified`` flag) is written into ``resolved_distribution`` so + an immutable revision always records exactly which artifact produced its + data. A condition whose distribution cannot be resolved keeps ``None`` and + is rejected by validation rather than silently frozen. + """ + timestamp = _now(now) + frozen_conditions: list[StudyCondition] = [] + for condition in protocol.conditions: + distribution = _resolve_distribution(condition, resolver) + frozen_conditions.append( + condition.model_copy( + update={ + "resolved_distribution": ( + distribution.to_frozen(resolved_at=timestamp) + if distribution.found + else None + ) + } + ) + ) + return protocol.model_copy(update={"conditions": frozen_conditions}) + + +def _check_environment(protocol: StudyProtocolV1) -> list[ValidationError]: + errors: list[ValidationError] = [] + environment = protocol.environment_requirements + + if not environment.expected_protocol_version: + errors.append( + ValidationError( + code=ValidationReasonCode.ENVIRONMENT_PROTOCOL_VERSION_MISSING, + field="environment_requirements.expected_protocol_version", + message="an expected ACP protocol version is required", + ) + ) + + for index, required in enumerate(environment.required_capabilities): + capability = required.capability + field = f"environment_requirements.required_capabilities[{index}]" + if capability not in _KNOWN_CAPABILITIES: + errors.append( + ValidationError( + code=ValidationReasonCode.UNKNOWN_REQUIRED_CAPABILITY, + field=field, + severity=ValidationSeverity.NEEDS_REVIEW, + message=( + f"required capability {capability!r} is not in the known " + "capability vocabulary" + ), + ) + ) + elif required.require_state not in _KNOWN_CAPABILITY_STATES: + errors.append( + ValidationError( + code=ValidationReasonCode.UNKNOWN_REQUIRED_CAPABILITY, + field=f"{field}.require_state", + severity=ValidationSeverity.NEEDS_REVIEW, + message=( + f"required state {required.require_state!r} is not a known " + "capability state" + ), + ) + ) + + if isinstance(environment.host_kind, ExplicitUnknown): + errors.append( + ValidationError( + code=ValidationReasonCode.UNKNOWN_ENVIRONMENT_REQUIREMENT, + field="environment_requirements.host_kind", + severity=ValidationSeverity.NEEDS_REVIEW, + message="host_kind is explicitly unknown and must be resolved", + ) + ) + + return errors + + +def _check_retention(protocol: StudyProtocolV1) -> list[ValidationError]: + """Reject retention policies that are not supported to publish. + + The supported policy retains uploaded data according to consent. A + participant-selected ``DELETE_ALL`` is not authorable: advertising a + deletion policy that is not the executed one is misleading. + """ + if protocol.privacy_policy.retention_action == RetentionAction.DELETE_ALL: + return [ + ValidationError( + code=ValidationReasonCode.RETENTION_UNSUPPORTED, + field="privacy_policy.retention_action", + message=( + "DELETE_ALL is not a supported published retention policy; " + "retain uploaded data according to consent" + ), + ) + ] + return [] + + +def _check_completion(protocol: StudyProtocolV1) -> list[ValidationError]: + completion = protocol.completion + if completion.policy == CompletionPolicyKind.TARGET_CAPACITY and ( + completion.target_enrollments is None or completion.target_enrollments <= 0 + ): + return [ + ValidationError( + code=ValidationReasonCode.COMPLETION_TARGET_INVALID, + field="completion.target_enrollments", + message="TARGET_CAPACITY completion requires a positive target", + ) + ] + return [] + + +def _check_protocol( + protocol: StudyProtocolV1, + resolver: Optional[DistributionResolver], + now: datetime, + *, + actor_is_admin: bool = False, +) -> list[ValidationError]: + errors: list[ValidationError] = [] + + if protocol.schema_version not in _VALID_SCHEMA_VERSIONS: + errors.append( + ValidationError( + code=ValidationReasonCode.UNSUPPORTED_SCHEMA_VERSION, + field="schema_version", + message=( + f"unsupported schema_version {protocol.schema_version!r}; " + f"expected one of {sorted(_VALID_SCHEMA_VERSIONS)}" + ), + ) + ) + + errors.extend(_check_conditions(protocol)) + errors.extend(_check_schedule(protocol, now)) + errors.extend(_check_assignment(protocol)) + errors.extend(_check_retention(protocol)) + errors.extend(_check_environment(protocol)) + errors.extend(_check_completion(protocol)) + errors.extend(_check_distributions(protocol, resolver, actor_is_admin)) + return errors + + +def validate_protocol( + document: Union[StudyProtocolV1, Mapping[str, Any]], + *, + distribution_resolver: Optional[DistributionResolver] = None, + actor_is_admin: bool = False, + now: Optional[datetime] = None, +) -> list[ValidationError]: + """Return every reason ``document`` cannot be published. + + An empty list means the document is publishable. A raw mapping is scanned + for forbidden identifiers/credentials *before* schema parsing so leakage is + always reported with a typed reason even when the extra keys also violate + the schema. + + ``distribution_resolver`` resolves each condition's ``distribution_id`` to + its release/verification view. ``actor_is_admin`` selects the severity of a + ``DISTRIBUTION_UNVERIFIED`` reason: a hard error for a researcher, a warning + for an administrator. + """ + if isinstance(document, StudyProtocolV1): + protocol: Optional[StudyProtocolV1] = document + raw: Any = document.model_dump(mode="json") + elif isinstance(document, Mapping): + protocol = None + raw = dict(document) + else: + return [ + ValidationError( + code=ValidationReasonCode.SCHEMA_INVALID, + field="", + message="protocol must be a StudyProtocolV1 or a mapping", + ) + ] + + errors = _scan_document(raw) + + if protocol is None: + try: + protocol = StudyProtocolV1.model_validate(raw) + except PydanticValidationError as error: + errors.append(_schema_error(error)) + return errors + + errors.extend( + _check_protocol( + protocol, + distribution_resolver, + _now(now), + actor_is_admin=actor_is_admin, + ) + ) + return errors + + +def scan_document_safety( + document: Union[StudyProtocolV1, Mapping[str, Any]] +) -> list[ValidationError]: + """Return only the forbidden-identifier/credential findings for a document. + + Used by draft creation, which must never persist a document that leaks a + participant/session/account identifier or a credential even though the draft + is not yet publishable. + """ + if isinstance(document, StudyProtocolV1): + raw: Any = document.model_dump(mode="json") + elif isinstance(document, Mapping): + raw = document + else: + return [] + return _scan_document(raw) + + +def blocking_errors(errors: list[ValidationError]) -> list[ValidationError]: + """The subset of ``errors`` that must block publication.""" + return [ + error for error in errors if error.severity != ValidationSeverity.WARNING + ] + + +def warnings(errors: list[ValidationError]) -> list[ValidationError]: + """The informational, non-blocking subset of ``errors``.""" + return [error for error in errors if error.severity == ValidationSeverity.WARNING] + + +def is_publishable(errors: list[ValidationError]) -> bool: + """Whether a validation result permits publication. + + ``ERROR`` and ``NEEDS_REVIEW`` both block publication (the latter asks for + human review rather than an automatic fix); ``WARNING`` is informational and + does not block. + """ + return not blocking_errors(errors) + + +def normalized_condition_weights(protocol: StudyProtocolV1) -> dict[str, float]: + """Deterministic ``w_i / sum(w)`` map for a valid protocol. + + Raises :class:`ProtocolValidationError` when the weights do not form a + normalizable positive total, so a caller can never divide by zero or + silently drop a condition. + """ + weights = { + condition.condition_id: condition.weight for condition in protocol.conditions + } + total = sum(weight for weight in weights.values() if weight is not None) + if not weights or not math.isfinite(total) or total <= 0: + raise ProtocolValidationError("condition weights do not sum to a positive total") + return { + condition_id: weight / total + for condition_id, weight in weights.items() + } diff --git a/src/research/telemetry/__init__.py b/src/research/telemetry/__init__.py new file mode 100644 index 0000000..c14ff58 --- /dev/null +++ b/src/research/telemetry/__init__.py @@ -0,0 +1,73 @@ +"""Canonical telemetry schema and privacy filter (Issue 06). + +One source-neutral event vocabulary that preserves provenance and enforces data +minimization before durable storage. + +Public surface: + +* :mod:`research.telemetry.enums` - canonical event types, sources, field + classes, policy actions, coverage states, canonical fidelity and lifecycle. +* :mod:`research.telemetry.models` - the immutable ``CanonicalEventV1`` + envelope and its sub-objects. +* :mod:`research.telemetry.builder` - event construction with a fresh id and a + strictly-increasing per-emitter sequence. +* :mod:`research.telemetry.validation` - typed envelope validation. +* :mod:`research.telemetry.privacy` - field classification and the redaction + engine that runs before any spool/log/retry/export. +* :mod:`research.telemetry.normalization` - ACP/IDE/legacy source normalizers + and the optional adapter enrichment interface. +* :mod:`research.telemetry.schema` - the language-neutral JSON Schema. + +The package never imports ``App``, FastAPI, or SQLAlchemy; it is the shared +contract that Issues 07-12 consume. +""" + +from .builder import EventBuilder, SequenceAllocator, build_event +from .enums import ( + CanonicalEventType, + CanonicalFidelity, + CoverageState, + EventSource, + FieldClass, + LifecycleState, + PolicyAction, +) +from .models import ( + CanonicalEventV1, + Correlations, + Coverage, + EventMetrics, + PrivacySummary, + Provenance, +) +from .privacy import PrivacyPolicy, filter_event, filter_payload +from .validation import ( + TelemetryValidationCode, + TelemetryValidationError, + validate_canonical_event, +) + +__all__ = [ + "CanonicalEventType", + "CanonicalEventV1", + "CanonicalFidelity", + "Correlations", + "Coverage", + "CoverageState", + "EventBuilder", + "EventMetrics", + "EventSource", + "FieldClass", + "LifecycleState", + "PolicyAction", + "PrivacyPolicy", + "PrivacySummary", + "Provenance", + "SequenceAllocator", + "TelemetryValidationCode", + "TelemetryValidationError", + "build_event", + "filter_event", + "filter_payload", + "validate_canonical_event", +] diff --git a/src/research/telemetry/adapters.py b/src/research/telemetry/adapters.py new file mode 100644 index 0000000..3d0dc30 --- /dev/null +++ b/src/research/telemetry/adapters.py @@ -0,0 +1,471 @@ +"""Server-authorized adapters feeding legacy producers into the canonical stream. + +The inference relay, the self-reporting runtime and the OTel span endpoint +observe agent activity that used to be written to the legacy ``agent_event`` +table. For a *research-bound* task (an explicit ``research_session_id`` / +``enrollment_id`` / ``study_id`` on ``agent_task``) those observations +are now built as canonical events and persisted through the one ingestion writer +(:func:`ingest_events_for_context`), so the dashboards read the canonical event +authority and there is no parallel canonical writer. + +A task with no research binding keeps the legacy operational path: ``agent_event`` +is retained for genuinely non-research operational data and this adapter returns +``None`` so the caller performs its legacy write. Nothing is fabricated: a fact +with no exact canonical equivalent is preserved as an explicit unknown (with its +original kind in ``payload.legacy_kind`` and provenance), never reclassified, and +a missing measurement is never turned into zero. + +For a research-bound task there is **no silent legacy fallback** (ISSUE-07): the +adapter always returns a result, and a canonical write that fails retryably or +is refused is reported to the caller so the producer keeps the batch instead of +recording it in the legacy table. +""" + +from __future__ import annotations + +import logging +import uuid +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Mapping, Optional, Sequence + +from sqlalchemy import text + +from research.participants import identity as identity_store +from research.runtime.sessions import store as session_store +from database import crud +from research.telemetry.builder import EventBuilder +from research.telemetry.content_policy import resolve_study_content_policy +from research.telemetry.enums import CanonicalEventType, EventSource, FieldClass, PolicyAction +from research.telemetry.ingestion.models import ( + IngestionContext, + TelemetryBatchAckV1, +) +from research.telemetry.ingestion.service import ingest_events_for_context +from research.telemetry.ingestion.store import SqlAlchemyIngestionStore +from research.telemetry.models import Correlations, Coverage, EventMetrics +from research.telemetry.privacy import PrivacyPolicy, classify_field, filter_event + +__all__ = [ + "CanonicalIngestionFailed", + "CanonicalRecordResult", + "LegacyFact", + "RELAY_NORMALIZER_VERSION", + "record_legacy_facts", + "research_bound", +] + +RELAY_NORMALIZER_VERSION = "relay-adapter-v1" + +#: Legacy producer kind -> canonical event type. A kind with no exact canonical +#: equivalent is preserved as ``unknown_source_event`` (the builder records the +#: original kind), never reclassified into a nearby concept. +_KIND_TO_TYPE: dict[str, CanonicalEventType] = { + "model_call": CanonicalEventType.AGENT_MESSAGE_COMPLETED, + "tool_call": CanonicalEventType.TOOL_COMPLETED, + "tool_failed": CanonicalEventType.TOOL_FAILED, + # NOTE: no "observation" entry on purpose: unmapped kinds keep their raw + # name so the builder stamps unknown_event_type instead of a marker-less + # unknown. + # Request-side counterparts of the completion kinds above. Mapping them to + # the matching start/created canonical types (instead of leaving them + # unknown) preserves the request→completion pairing without double + # counting: token/step aggregation only ever reads the completion side. + "model_request": CanonicalEventType.AGENT_MESSAGE_STARTED, + "tool_request": CanonicalEventType.TOOL_CREATED, + "run_started": CanonicalEventType.AGENT_RUN_STARTED, + "run_completed": CanonicalEventType.AGENT_RUN_COMPLETED, + # Permission request/decision outcomes. The decision value (accepted / + # rejected / cancelled / unavailable) rides in the payload; the kind + # mapping itself never invents it. + "permission_requested": CanonicalEventType.PERMISSION_REQUESTED, + "permission_decided": CanonicalEventType.PERMISSION_DECIDED, +} + + +class CanonicalIngestionFailed(Exception): + """A research-bound canonical write did not commit. + + Raised so a route can map the failure to a truthful HTTP result instead of + reporting success or silently writing the legacy table (ISSUE-07). + """ + + def __init__( + self, + *, + reason: str, + message: str = "", + retryable: bool = True, + ack: Optional[TelemetryBatchAckV1] = None, + ) -> None: + super().__init__(message or reason) + self.reason = reason + self.message = message or reason + self.retryable = retryable + self.ack = ack + + +@dataclass(frozen=True) +class CanonicalRecordResult: + """Outcome of one internal canonical write (ISSUE-07). + + ``written`` is True only after the ingestion writer committed. A research + bound caller must treat any non-written result as a failure: ``retryable`` + marks a transient failure (keep and re-send the batch), otherwise the facts + were terminally refused. + """ + + written: bool + retryable: bool = False + reason: str = "OK" + message: str = "" + ack: Optional[TelemetryBatchAckV1] = None + + @property + def ingested_event_ids(self) -> tuple[uuid.UUID, ...]: + if self.ack is None: + return () + return tuple( + entry.event_id + for entry in (self.ack.accepted + self.ack.duplicate) + if entry.event_id is not None + ) + + +@dataclass(frozen=True) +class LegacyFact: + """One already-sanitized legacy observation, ready to canonicalize. + + ``payload`` must contain only structural metadata (no prompt/source/output/ + diff/reasoning content): content stays behind the legacy consent gate. + """ + + kind: str + occurred_at: datetime + payload: dict[str, Any] = field(default_factory=dict) + metrics: EventMetrics = field(default_factory=EventMetrics) + coverage: Coverage = field(default_factory=Coverage) + correlations: Correlations = field(default_factory=Correlations) + source_event_id: Optional[str] = None + emitter_id: str = "relay" + + +def research_bound(task: Any) -> bool: + """Whether a task carries the explicit research binding. + + Either canonical attribution id is sufficient: ``enrollment_id`` alone (a + binding without an unambiguous session) must still use the canonical + authority and must never fall back to the legacy table. This is pinned by + ``test_agent_event_boundary.py``. + """ + return ( + getattr(task, "research_session_id", None) is not None + or getattr(task, "enrollment_id", None) is not None + ) + + +#: Plain metadata classes a policy may exclude; see ``_policy_payload``. +#: The server's own markers (``legacy_kind``, ``upstream_status``) are SYSTEM +#: (``SYSTEM_KEY_EXACT``), so a METRICS-only policy keeps them and the +#: dashboards that read ``legacy_kind`` do not report zero. +_EXCLUDABLE_METADATA = frozenset({FieldClass.SYSTEM, FieldClass.BEHAVIORAL}) + + +def _policy_payload( + payload: Mapping[str, Any], policy: Optional[PrivacyPolicy] +) -> dict[str, Any]: + """``payload`` without the scalar SYSTEM/BEHAVIORAL fields ``policy`` excludes. + + Clients filter their events before upload, so ingestion refuses any event + the study policy would still change. These events are built here on the + server, so they are built compliant instead: a policy that excludes agent + activity (e.g. usage and timings only) would otherwise refuse every + self-report and model call, losing the timings and token counts the study + does collect. Content, secrets and code metadata are left for the ingestion + check, which refuses them rather than stripping (ISSUE-01); only secrets + inside consented content are dropped first (see ``build_legacy_events``). + """ + if policy is None: + return dict(payload) + kept: dict[str, Any] = {} + for key, value in payload.items(): + if not isinstance(value, (Mapping, list, tuple)): + field_class = classify_field(key, value) + # Only what the engine itself would drop; a blocked class is left + # for ingestion to refuse. + if field_class in _EXCLUDABLE_METADATA and policy.action_for(field_class) is PolicyAction.DROP: + continue + kept[key] = value + return kept + + +def build_legacy_events( + task: Any, + facts: Sequence[LegacyFact], + policy: Optional[PrivacyPolicy] = None, + *, + first_sequence: Optional[int] = None, +) -> list: + """Build canonical events for a research-bound task, in order. + + With ``policy``, plain metadata fields it excludes are left out of each + payload (see ``_policy_payload``). Under a content-capture policy with + consent, each event is then filtered like a client's (``filter_event``): a + secret-shaped value inside the consented content is dropped instead of + refusing the whole fact. A model call that quotes "Bearer tokens" in its + conversation would otherwise lose its usage and timings, and so would every + later call of that chat, which repeats the history. Content the policy does + not allow, or a class it blocks, is still refused by ingestion (ISSUE-01). + + ``(research_session_id, emitter_id, emitter_sequence)`` is a unique key in + the store, so sequences must never be reused within a session+emitter. + Callers that persist must reserve ``len(facts)`` indexes up front (see + ``record_legacy_facts``) and pass the reserved base as ``first_sequence``; + deriving from ``task.next_event_index`` without reserving replays the same + sequences on every call and every later batch is rejected as an integrity + conflict. When ``first_sequence`` is omitted the legacy counter-derived + numbering is kept for backward compatibility (tests, dry runs). + + The relay adapts spans from many tasks in one session, but the store's + uniqueness key is ``(session, emitter, sequence)``: sharing one emitter + across tasks would collide independent streams. Each task's facts are + therefore their own emitter namespace (``":"``), matching + the documented model that independent emitters each own their sequence. + """ + builder = EventBuilder() + task_namespace = str(getattr(task, "task_id", "unbound")) + # The run scopes the trace for facts that didn't set one explicitly. + # Read after the caller-supplied context: record_legacy_facts self-heals + # a missing run id before building, so this is never stale there. + agent_run_id = getattr(task, "external_run_id", None) + if first_sequence is None: + sequence = int(getattr(task, "next_event_index", 0) or 0) + else: + # The reserved base is the 0-based legacy event index; canonical + # emitter sequences are 1-based continuations of the same counter, + # matching the historical counter-derived numbering (first event of a + # fresh task is sequence 1, never 0 which validation rejects). + sequence = int(first_sequence) + events = [] + for fact in facts: + sequence += 1 + mapped = _KIND_TO_TYPE.get(fact.kind) + # An unmapped kind keeps its raw name so the builder stamps + # ``unknown_event_type`` (and NEEDS_REVIEW coverage) instead of a + # marker-less unknown that is useless for forensics. + event_type = mapped if mapped is not None else fact.kind + # Default the trace/span handles when the producer didn't set them: + # the run scopes the trace and the source event id is the span. + # Anything explicitly reported wins; nothing is invented beyond that. + base_corr = fact.correlations or Correlations() + corr_update: dict[str, Any] = {} + if base_corr.trace_id is None and agent_run_id is not None: + corr_update["trace_id"] = agent_run_id + if base_corr.span_id is None and fact.source_event_id is not None: + corr_update["span_id"] = fact.source_event_id + correlations = ( + base_corr.model_copy(update=corr_update) if corr_update else base_corr + ) + built = builder.build( + emitter_id=f"{fact.emitter_id}:{task_namespace}", + event_type=event_type, + source=EventSource.RELAY, + occurred_at=fact.occurred_at, + normalizer_version=RELAY_NORMALIZER_VERSION, + payload=_policy_payload({**fact.payload, "legacy_kind": fact.kind}, policy), + metrics=fact.metrics, + coverage=fact.coverage, + correlations=correlations, + source_event_id=fact.source_event_id, + study_id=getattr(task, "study_id", None), + enrollment_id=getattr(task, "enrollment_id", None), + research_session_id=getattr(task, "research_session_id", None), + agent_run_id=getattr(task, "external_run_id", None), + emitter_sequence=sequence, + ) + if policy is not None and policy.content_allowed and policy.consent_active: + filtered = filter_event(built, policy) + if not filtered.summary.blocked: + built = filtered.event + events.append(built) + return events + + +def _kill_switch_check(db: Any, task: Any): + """Scope the operator kill switch to the task's study/enrollment.""" + from research.analysis.operations import store as operations_store + + return operations_store.db_kill_switch_check( + db, + study_id=getattr(task, "study_id", None), + enrollment_id=getattr(task, "enrollment_id", None), + ) + + +def existing_source_event_ids(db: Any, task: Any, source_event_ids: Sequence[str]) -> set[str]: + """Runtime event ids already stored canonically for ``task``'s run. + + The self-report ingest dedupes retries by the runtime's own event id. For a + research-bound task those rows live in ``research_event`` (never in the + legacy table), so this is the lookup that makes a retried batch report + duplicates instead of storing every event twice. Advisory: a lookup + failure returns nothing and the batch proceeds. + """ + run_id = getattr(task, "external_run_id", None) + ids = [str(value) for value in source_event_ids if value] + if not run_id or not ids: + return set() + begin_nested = getattr(db, "begin_nested", None) + savepoint = begin_nested() if callable(begin_nested) else None + try: + # The builder stores the runtime's id under ``provenance``. + rows = db.execute( + text( + "SELECT envelope_json -> 'provenance' ->> 'source_event_id' AS source_event_id " + "FROM research_event " + "WHERE agent_run_id = :run_id " + "AND envelope_json -> 'provenance' ->> 'source_event_id' = ANY(:ids)" + ), + {"run_id": str(run_id), "ids": ids}, + ).all() + if savepoint is not None: + savepoint.commit() + except Exception: # noqa: BLE001 - advisory lookup + # Roll back only the savepoint, so a failed lookup never aborts the + # surrounding ingest transaction. + if savepoint is not None: + savepoint.rollback() + return set() + return {str(row.source_event_id) for row in rows if row.source_event_id} + + +def record_legacy_facts( + db: Any, + *, + task: Any, + facts: Sequence[LegacyFact], + first_sequence: Optional[int] = None, +) -> Optional[CanonicalRecordResult]: + """Persist legacy facts through the canonical ingestion writer. + + Returns ``None`` when the task has no research binding, so the caller keeps + its legacy operational write. For a research-bound task it always returns a + result: the caller must not fall back to ``agent_event`` even when the + canonical write failed (ISSUE-07). Never raises. + + A caller that already reserved ``len(facts)`` task event indexes (the + self-report ingest does, for its legacy rows) passes the reserved base as + ``first_sequence``; reserving a second block would open a sequence gap + before every batch. + """ + if task is None or not facts or not research_bound(task): + return None + try: + enrollment_row = ( + identity_store.get_enrollment(db, task.enrollment_id, for_update=True) + if getattr(task, "enrollment_id", None) is not None + else None + ) + session_row = ( + session_store.get_session(db, task.research_session_id, for_update=True) + if getattr(task, "research_session_id", None) is not None + else None + ) + if enrollment_row is None or session_row is None: + return CanonicalRecordResult( + written=False, + retryable=True, + reason="RESEARCH_CONTEXT_UNAVAILABLE", + message=( + "the task's enrollment/session rows are unavailable; " + "the batch must be retried, never written to the legacy table" + ), + ) + enrollment = identity_store.row_to_enrollment(enrollment_row) + session = session_store.row_to_session(session_row) + + # Self-heal the run correlation: tasks created before run ids were + # minted (or through paths that skip them) would otherwise stamp + # agent_run_id=None, leaving their events unattributable in every + # dashboard join. Minting here is idempotent — tasks that already + # have one keep it — and stays inside the same transaction. + if getattr(task, "external_run_id", None) in (None, ""): + task.external_run_id = uuid.uuid4().hex + flush = getattr(db, "flush", None) + if callable(flush): + try: + flush() + except Exception: + db.rollback() + return CanonicalRecordResult( + written=False, + retryable=True, + reason="STORE_UNAVAILABLE", + message="could not persist the task run correlation; retry the batch", + ) + + # The study policy is the single authority for content storage on this + # boundary too (ISSUE-01/ISSUE-07). Missing/malformed policy denies + # content; the ingestion writer rejects the offending facts instead of + # storing them. + decision = resolve_study_content_policy( + db, + account_id=getattr(task, "owner_user_id", None), + study_id=task.study_id, + enrollment_id=task.enrollment_id, + ) + policy = decision.policy or PrivacyPolicy.default() + + context = IngestionContext( + study_id=task.study_id, + enrollment_id=task.enrollment_id, + research_session_id=task.research_session_id, + revocation_epoch=enrollment.revocation_epoch, + ) + # Reserve the emitter sequences atomically before building: without + # this, every call replays sequences from the (never advanced) task + # counter and every batch after the first is rejected as an integrity + # conflict. Burned indexes on failed writes surface as ordinary gaps. + if first_sequence is None: + first_sequence = crud.reserve_agent_event_indexes( + db, getattr(task, "task_id", None), len(facts) + ) + events = build_legacy_events(task, facts, policy, first_sequence=first_sequence) + ack = ingest_events_for_context( + context=context, + enrollment=enrollment, + session=session, + events=events, + store=SqlAlchemyIngestionStore(db), + kill_switch_check=_kill_switch_check(db, task), + privacy_policy=policy, + ) + if ack.retryable: + return CanonicalRecordResult( + written=False, + retryable=True, + reason="RETRYABLE", + message="canonical ingestion returned retryable events", + ack=ack, + ) + if ack.rejected and not (ack.accepted or ack.duplicate): + return CanonicalRecordResult( + written=False, + retryable=False, + reason="REJECTED", + message="canonical ingestion rejected the batch", + ack=ack, + ) + return CanonicalRecordResult(written=True, reason="OK", ack=ack) + except Exception as error: # noqa: BLE001 - research-bound writes must not raise + logging.warning( + "[relay-adapter] canonical write failed for task %s — %s", + getattr(task, "task_id", None), + error, + ) + return CanonicalRecordResult( + written=False, + retryable=True, + reason="STORE_UNAVAILABLE", + message="canonical ingestion store is unavailable; retry the batch", + ) diff --git a/src/research/telemetry/builder.py b/src/research/telemetry/builder.py new file mode 100644 index 0000000..25d6f43 --- /dev/null +++ b/src/research/telemetry/builder.py @@ -0,0 +1,188 @@ +"""Canonical event construction and per-emitter sequencing. + +The builder owns two things the source mappers must not invent: + +* a fresh, immutable ``event_id``; +* a strictly-increasing ``emitter_sequence`` **per emitter** (independent + emitters each start their own sequence; there is no fake global counter). + +It never derives cross-process latency: ``latency_ms`` is only copied from an +explicit caller-provided metric. It also does not redact; callers must pass the +result through :func:`research.telemetry.privacy.engine.filter_event` before the +event reaches a spool, log, retry queue, or export. +""" + +from __future__ import annotations + +import threading +import uuid +from typing import TYPE_CHECKING, Any, Mapping, Optional + +from .enums import ( + CANONICAL_SCHEMA_VERSION, + CanonicalEventType, + CanonicalFidelity, + CoverageState, + EventSource, + LifecycleState, +) +from .models import ( + CanonicalEventV1, + Correlations, + Coverage, + EventMetrics, + PrivacySummary, + Provenance, +) + +if TYPE_CHECKING: + from datetime import datetime + from uuid import UUID + + +class SequenceAllocator: + """Allocates strictly-increasing sequences independently per emitter.""" + + def __init__(self, start: int = 1) -> None: + self._start = start + self._last: dict[str, int] = {} + # The research proxy numbers events from both forwarding threads and, + # when it stops, from its main thread: no two may share a sequence. + self._lock = threading.Lock() + + def next(self, emitter_id: str) -> int: + """Return the next sequence for ``emitter_id`` (starting at ``start``).""" + with self._lock: + value = self._last.get(emitter_id, self._start - 1) + 1 + self._last[emitter_id] = value + return value + + def current(self, emitter_id: str) -> int: + """Return the last allocated sequence for ``emitter_id`` (0 if none).""" + with self._lock: + return self._last.get(emitter_id, self._start - 1) + + +def canonical_event_type(raw: Any) -> tuple[str, Optional[str]]: + """Return ``(canonical_type, unknown_original)`` for a raw event type.""" + if isinstance(raw, CanonicalEventType): + return raw.value, None + text = str(raw) + try: + return CanonicalEventType(text).value, None + except ValueError: + return CanonicalEventType.UNKNOWN_SOURCE_EVENT.value, text + + +def canonical_source(raw: Any) -> tuple[str, Optional[str]]: + """Return ``(canonical_source, unknown_original)`` for a raw source.""" + if isinstance(raw, EventSource): + return raw.value, None + text = str(raw) + try: + return EventSource(text.strip().lower()).value, None + except ValueError: + return text, text + + +def canonical_lifecycle(raw: Any) -> tuple[Optional[str], Optional[str]]: + """Return ``(canonical_lifecycle, unknown_original)`` for a raw state.""" + if raw is None: + return None, None + if isinstance(raw, LifecycleState): + return raw.value, None + text = str(raw) + try: + return LifecycleState(text.strip().lower()).value, None + except ValueError: + return text, text + + +class EventBuilder: + """Builds immutable :class:`CanonicalEventV1` envelopes.""" + + def __init__(self, allocator: Optional[SequenceAllocator] = None) -> None: + self.allocator = allocator or SequenceAllocator() + + def build( + self, + *, + emitter_id: str, + event_type: Any, + source: Any, + occurred_at: datetime, + normalizer_version: str, + payload: Optional[Mapping[str, Any]] = None, + fidelity: CanonicalFidelity = CanonicalFidelity.EXACT, + adapter_version: Optional[str] = None, + source_event_id: Optional[str] = None, + evidence_digest: Optional[str] = None, + metrics: Optional[EventMetrics] = None, + coverage: Optional[Coverage] = None, + privacy: Optional[PrivacySummary] = None, + lifecycle_state: Any = None, + correlations: Optional[Correlations] = None, + monotonic_ns: Optional[int] = None, + study_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + research_session_id: Optional[UUID] = None, + agent_run_id: Optional[str] = None, + event_id: Optional[UUID] = None, + emitter_sequence: Optional[int] = None, + ) -> CanonicalEventV1: + """Build one canonical event with a fresh id and allocated sequence.""" + canonical_type, unknown_event_type = canonical_event_type(event_type) + canonical_source_value, unknown_source = canonical_source(source) + lifecycle, unknown_lifecycle = canonical_lifecycle(lifecycle_state) + + resolved_coverage = coverage or Coverage() + if unknown_event_type or unknown_source or unknown_lifecycle: + resolved_coverage = Coverage( + state=CoverageState.NEEDS_REVIEW, + reason="unrecognized input preserved for review", + capability=resolved_coverage.capability, + ) + + return CanonicalEventV1( + event_id=event_id or uuid.uuid4(), + schema_version=CANONICAL_SCHEMA_VERSION, + event_type=canonical_type, + source=canonical_source_value, + study_id=study_id, + enrollment_id=enrollment_id, + research_session_id=research_session_id, + agent_run_id=agent_run_id, + occurred_at=occurred_at, + monotonic_ns=monotonic_ns, + emitter_id=emitter_id, + emitter_sequence=( + emitter_sequence + if emitter_sequence is not None + else self.allocator.next(emitter_id) + ), + correlations=correlations or Correlations(), + lifecycle_state=lifecycle, + payload=dict(payload or {}), + metrics=metrics or EventMetrics(), + privacy=privacy or PrivacySummary(), + provenance=Provenance( + source=canonical_source_value, + source_event_id=source_event_id, + normalizer_version=normalizer_version, + adapter_version=adapter_version, + fidelity=fidelity, + evidence_digest=evidence_digest, + ), + coverage=resolved_coverage, + unknown_event_type=unknown_event_type, + unknown_source=unknown_source, + unknown_lifecycle_state=unknown_lifecycle, + ) + + +_DEFAULT_ALLOCATOR = SequenceAllocator() + + +def build_event(**kwargs: Any) -> CanonicalEventV1: + """Build an event using the process-wide default sequence allocator.""" + return EventBuilder(_DEFAULT_ALLOCATOR).build(**kwargs) diff --git a/src/research/telemetry/chat_lifecycle.py b/src/research/telemetry/chat_lifecycle.py new file mode 100644 index 0000000..7699830 --- /dev/null +++ b/src/research/telemetry/chat_lifecycle.py @@ -0,0 +1,85 @@ +"""Chat-lifecycle and Revise telemetry contract shared by producers and readers. + +The proxy-side ACP normalizer (:mod:`research.telemetry.normalization.generic_acp`), +the participant proxy and the study analytics read model all use these literals, +so they stay additive: renaming one breaks already-stored events. + +The payload keys are chosen so the privacy classifier keeps them as metadata +(``acp_method``, ``end_reason`` and ``selected_option_id`` are BEHAVIORAL; keys +ending in ``_count`` are SYSTEM); an unrecognised key would be classed CONTENT +and redacted under a metadata-only study. + +No new canonical event type is involved: a chat start is an +``interaction.started`` and a chat end an ``interaction.completed`` with +lifecycle ``completed``. A user interrupt (``session/cancel``) stays an +``interaction.completed`` without a lifecycle state, which is how analytics keeps +chat ends out of the cancel count. +""" + +from __future__ import annotations + +__all__ = [ + "ACP_METHOD_KEY", + "CANCEL_METHOD", + "CHAT_END_METHODS", + "CHAT_START_METHODS", + "END_REASON_AGENT_EXITED", + "END_REASON_HOST_CLOSED", + "END_REASON_KEY", + "END_REASON_SESSION_STALE", + "END_REASON_SIGNAL_TERMINATED", + "PROXY_END_REASONS", + "REVISED_DECISION", + "REVISE_OPTION_ID", + "SELECTED_OPTION_ID_KEY", +] + +#: Payload key naming the ACP method an event was derived from. +ACP_METHOD_KEY = "acp_method" + +#: Payload key on the proxy's final event saying why the chat's process ended. +END_REASON_KEY = "end_reason" + +#: Payload key on ``permission.decided`` carrying the option the user picked. +SELECTED_OPTION_ID_KEY = "selected_option_id" + +#: ACP methods that start or reattach to a chat, mapped to the start kind +#: analytics reports. +CHAT_START_METHODS: dict[str, str] = { + "session/new": "new", + "session/fork": "fork", + "session/load": "load", + "session/resume": "resume", +} + +#: ACP methods with which the client ends a chat. +CHAT_END_METHODS = frozenset({"session/close", "session/delete"}) + +#: The ACP user interrupt; its ``interaction.completed`` is the only cancel. +CANCEL_METHOD = "session/cancel" + +#: The host closed the proxy's stdin (IntelliJ releases a chat's agent process +#: when the chat is deleted from its history, and on IDE shutdown). +END_REASON_HOST_CLOSED = "host_closed" +#: The agent process exited on its own. +END_REASON_AGENT_EXITED = "agent_exited" +#: The proxy stopped because its research session went stale. +END_REASON_SESSION_STALE = "session_stale" +#: The proxy received SIGTERM/SIGINT. +END_REASON_SIGNAL_TERMINATED = "signal_terminated" + +PROXY_END_REASONS = frozenset( + { + END_REASON_HOST_CLOSED, + END_REASON_AGENT_EXITED, + END_REASON_SESSION_STALE, + END_REASON_SIGNAL_TERMINATED, + } +) + +#: The permission option id the built-in agent offers for "Revise…". The proxy +#: records it as ``selected_option_id`` on a reject-kind decision. +REVISE_OPTION_ID = "revise" + +#: The decision the built-in agent self-reports for an accepted revision form. +REVISED_DECISION = "revised" diff --git a/src/research/telemetry/content_policy.py b/src/research/telemetry/content_policy.py new file mode 100644 index 0000000..9449286 --- /dev/null +++ b/src/research/telemetry/content_policy.py @@ -0,0 +1,206 @@ +"""Single server-side authority for content-storage permission (ISSUE-01). + +For a **research-bound** context the permission to persist prompts, tool +arguments/results and raw payloads is derived from the account's ACTIVE +enrollment plus the study's frozen telemetry policy. The legacy account +preference (``store_agent_content``) is *not* study consent and is only +consulted for genuinely non-research contexts, where there is no study policy +to enforce. + +The decision is fail-closed: + +* no participant mapping / no active enrollment / no study row -> deny; +* missing or malformed telemetry policy -> deny; +* policy without ``content_capture`` rights, or inactive consent -> deny; +* only an ACTIVE enrollment with accepted consent under a policy that grants + content capture allows content. + +Every persistence boundary (managed-agent ingest, proxy/plugin task creation, +legacy relay adapters and the ACP policy endpoints) calls this module so there +is exactly one place where the study decision is made. +""" + +from __future__ import annotations + +import logging +import uuid +from dataclasses import dataclass +from typing import Any, Mapping, Optional + +from database.db_schemas import Study as StudyRow +from research.participants import identity as identity_store +from research.participants.enums import EnrollmentStatus +from research.telemetry.privacy import PrivacyPolicy + +__all__ = [ + "ContentPolicyDecision", + "DENY_DATA_COLLECTION_OPTED_OUT", + "DENY_LEGACY_PREFERENCE", + "DENY_NO_ACTIVE_ENROLLMENT", + "DENY_NO_PARTICIPANT", + "DENY_POLICY_DENIES_CONTENT", + "DENY_POLICY_MALFORMED", + "DENY_STUDY_UNAVAILABLE", + "LEGACY_PREFERENCE", + "STUDY_POLICY", + "resolve_study_content_policy", +] + +STUDY_POLICY = "STUDY_POLICY" +LEGACY_PREFERENCE = "LEGACY_PREFERENCE" + +DENY_NO_PARTICIPANT = "NO_PARTICIPANT" +DENY_NO_ACTIVE_ENROLLMENT = "NO_ACTIVE_ENROLLMENT" +DENY_STUDY_UNAVAILABLE = "STUDY_UNAVAILABLE" +DENY_POLICY_MALFORMED = "POLICY_MALFORMED" +DENY_POLICY_DENIES_CONTENT = "POLICY_DENIES_CONTENT" +DENY_LEGACY_PREFERENCE = "LEGACY_PREFERENCE_DENIED" +# The account withdrew consent (``privacy.collection``); overrides study policy. +DENY_DATA_COLLECTION_OPTED_OUT = "DATA_COLLECTION_OPTED_OUT" + + +@dataclass(frozen=True) +class ContentPolicyDecision: + """One resolved content-storage permission and its audit reason.""" + + allowed: bool + reason: str + policy: Optional[PrivacyPolicy] = None + enrollment_id: Optional[uuid.UUID] = None + study_id: Optional[uuid.UUID] = None + policy_digest: Optional[str] = None + + +def _deny( + reason: str, + *, + policy: Optional[PrivacyPolicy] = None, + enrollment_id: Optional[uuid.UUID] = None, + study_id: Optional[uuid.UUID] = None, +) -> ContentPolicyDecision: + return ContentPolicyDecision( + allowed=False, + reason=reason, + policy=policy, + enrollment_id=enrollment_id, + study_id=study_id, + policy_digest=policy.policy_digest if policy is not None else None, + ) + + +def _active_enrollment( + db: Any, + *, + account_id: uuid.UUID, + study_id: uuid.UUID, + enrollment_id: Optional[uuid.UUID] = None, + for_update: bool = False, +): + participant = identity_store.get_participant_by_account(db, account_id) + if participant is None: + return None + candidates = [ + row + for row in identity_store.list_enrollments(db, participant.participant_id) + if row.status == EnrollmentStatus.ACTIVE.value + and getattr(row, "study_id", None) == study_id + ] + if enrollment_id is not None: + candidates = [ + row for row in candidates if row.enrollment_id == enrollment_id + ] + if len(candidates) != 1: + # Zero or ambiguous (>1) active enrollments: never guess a consent + # subject; the caller gets a fail-closed decision. + return None + row = candidates[0] + if for_update: + return identity_store.get_enrollment(db, row.enrollment_id, for_update=True) + return row + + +def resolve_study_content_policy( + db: Any, + *, + account_id: Optional[uuid.UUID], + study_id: uuid.UUID, + enrollment_id: Optional[uuid.UUID] = None, +) -> ContentPolicyDecision: + """Resolve content-storage permission from enrollment + frozen study policy. + + ``account_id`` is the authenticated account, ``study_id`` the frozen study + binding of the task/run, and the optional ``enrollment_id`` pins one exact + enrollment (a mismatch denies). Returns a decision; never raises for a + missing/broken subject or policy. + """ + if account_id is None: + return _deny(DENY_NO_PARTICIPANT, study_id=study_id) + + try: + enrollment = _active_enrollment( + db, + account_id=account_id, + study_id=study_id, + enrollment_id=enrollment_id, + ) + except Exception as error: # noqa: BLE001 - fail closed on store problems + logging.warning( + "[ContentPolicy] enrollment lookup failed for study %s: %s", + study_id, + error, + ) + return _deny(DENY_NO_ACTIVE_ENROLLMENT, study_id=study_id) + + if enrollment is None: + return _deny(DENY_NO_ACTIVE_ENROLLMENT, study_id=study_id) + + try: + study = db.get(StudyRow, study_id) + except Exception as error: # noqa: BLE001 - fail closed on store problems + logging.warning("[ContentPolicy] study lookup failed for %s: %s", study_id, error) + return _deny( + DENY_STUDY_UNAVAILABLE, + enrollment_id=enrollment.enrollment_id, + study_id=study_id, + ) + if study is None: + return _deny( + DENY_STUDY_UNAVAILABLE, + enrollment_id=enrollment.enrollment_id, + study_id=study_id, + ) + + config = getattr(study, "research_config_json", None) or {} + raw_policy = config.get("telemetry_policy") + if not isinstance(raw_policy, Mapping): + return _deny( + DENY_POLICY_MALFORMED, + enrollment_id=enrollment.enrollment_id, + study_id=study_id, + ) + + consent_active = ( + enrollment.status == EnrollmentStatus.ACTIVE.value + and getattr(enrollment, "consent_accepted_at", None) is not None + ) + try: + policy = PrivacyPolicy.from_study_policy( + dict(raw_policy), consent_active=consent_active + ) + except Exception as error: # noqa: BLE001 - malformed policy denies + logging.warning("[ContentPolicy] policy parse failed for study %s: %s", study_id, error) + return _deny( + DENY_POLICY_MALFORMED, + enrollment_id=enrollment.enrollment_id, + study_id=study_id, + ) + + allowed = bool(policy.content_allowed and policy.consent_active) + return ContentPolicyDecision( + allowed=allowed, + reason=STUDY_POLICY if allowed else DENY_POLICY_DENIES_CONTENT, + policy=policy, + enrollment_id=enrollment.enrollment_id, + study_id=study_id, + policy_digest=policy.policy_digest, + ) diff --git a/src/research/telemetry/enums.py b/src/research/telemetry/enums.py new file mode 100644 index 0000000..382e553 --- /dev/null +++ b/src/research/telemetry/enums.py @@ -0,0 +1,149 @@ +"""Closed vocabularies for the canonical telemetry schema (Issue 06). + +Every member must have a real producer (the IDE collector, the shared ACP/proxy +normalizer, or the server); a member with no producer is removed rather than +kept as an unreachable reserved token, because the vocabulary is the persisted/ +exported contract. Otherwise members are additive only: renaming a produced +member is a breaking change to already-stored events. + +Fidelity note: :class:`CanonicalFidelity` uses the **lowercase** telemetry-spec +values (``exact`` / ``normalized`` / ``inferred``). It is deliberately distinct +from the **uppercase** receipt fidelity in +:class:`research.compatibility.enums.Fidelity` (``EXACT`` / ``NORMALIZED`` / +``INFERRED``); the two must not be conflated. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = [ + "CANONICAL_SCHEMA_VERSION", + "CanonicalEventType", + "CanonicalFidelity", + "CoverageState", + "EventSource", + "FieldClass", + "LifecycleState", + "PolicyAction", +] + +#: The canonical telemetry envelope schema version. It is required on every +#: event and is never defaulted on ingestion. +CANONICAL_SCHEMA_VERSION = "1" + + +class CanonicalEventType(str, Enum): + """Canonical, vendor-independent event concepts. + + These are canonical concepts, not ACP method names: an ACP notification is + *normalized into* one of these, and an unknown construct becomes + :attr:`UNKNOWN_SOURCE_EVENT` rather than a new vendor-named type. + + Members are additive only. A member with no producer anywhere in the + platform (plugin IDE collector, shared ACP/proxy normalizer, server) is not + part of the vocabulary: it was removed rather than left as an unreachable + reserved token. + """ + + INTERACTION_STARTED = "interaction.started" + INTERACTION_COMPLETED = "interaction.completed" + + AGENT_RUN_STARTED = "agent.run.started" + AGENT_RUN_COMPLETED = "agent.run.completed" + + AGENT_MESSAGE_STARTED = "agent.message.started" + AGENT_MESSAGE_COMPLETED = "agent.message.completed" + + TOOL_CREATED = "tool.created" + TOOL_STARTED = "tool.started" + TOOL_COMPLETED = "tool.completed" + TOOL_FAILED = "tool.failed" + + PERMISSION_REQUESTED = "permission.requested" + PERMISSION_DECIDED = "permission.decided" + + PLAN_UPDATED = "plan.updated" + USAGE_UPDATED = "usage.updated" + + IDE_DOCUMENT_CHANGED = "ide.document.changed" + IDE_FILE_OPENED = "ide.file.opened" + IDE_FILE_SAVED = "ide.file.saved" + IDE_FILE_CLOSED = "ide.file.closed" + IDE_RUN_EXECUTED = "ide.run.executed" + + SYSTEM_AGENT_CRASHED = "system.agent.crashed" + SYSTEM_PROXY_ERROR = "system.proxy.error" + + AGENT_ERROR = "agent.error" + + UNKNOWN_SOURCE_EVENT = "unknown_source_event" + + +class EventSource(str, Enum): + """Which observation channel emitted the source event (lowercase values).""" + + ACP = "acp" + IDE = "ide" + LEGACY = "legacy" + # The backend inference relay observes provider model calls directly; it is + # a distinct observation channel from the ACP proxy and the IDE collector, + # so source ownership can prevent double counting the same action. + RELAY = "relay" + + +class FieldClass(str, Enum): + """Data classification used to decide whether a field may persist.""" + + SYSTEM = "SYSTEM" + BEHAVIORAL = "BEHAVIORAL" + CODE_METADATA = "CODE_METADATA" + CONTENT = "CONTENT" + SECRET = "SECRET" + + +class PolicyAction(str, Enum): + """Terminal privacy decision for one field (or the whole event).""" + + ALLOW = "ALLOW" + REDACT = "REDACT" + HASH = "HASH" + DROP = "DROP" + BLOCK = "BLOCK" + + +class CoverageState(str, Enum): + """Whether a measurement/observation is present, absent, or disputed. + + ``UNAVAILABLE`` means "the source does not expose it" and ``UNKNOWN`` means + "not observed"; neither is zero or false. + """ + + AVAILABLE = "AVAILABLE" + UNAVAILABLE = "UNAVAILABLE" + UNKNOWN = "UNKNOWN" + PARTIAL = "PARTIAL" + NEEDS_REVIEW = "NEEDS_REVIEW" + + +class CanonicalFidelity(str, Enum): + """How faithfully a canonical field represents its source observation. + + Lowercase by canonical-spec convention; distinct from the uppercase receipt + fidelity vocabulary in Issue 01. + """ + + EXACT = "exact" + NORMALIZED = "normalized" + INFERRED = "inferred" + + +class LifecycleState(str, Enum): + """Lifecycle phase of a lifecycled canonical event.""" + + PENDING = "pending" + STARTED = "started" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + UNKNOWN = "unknown" diff --git a/src/research/telemetry/ingestion/__init__.py b/src/research/telemetry/ingestion/__init__.py new file mode 100644 index 0000000..1f0f0e2 --- /dev/null +++ b/src/research/telemetry/ingestion/__init__.py @@ -0,0 +1,66 @@ +"""Idempotent canonical telemetry ingestion (Issue 09). + +Public surface: + +* :mod:`research.telemetry.ingestion.enums` - dispositions and typed ingest reason codes. +* :mod:`research.telemetry.ingestion.models` - ``TelemetryBatchRequestV1`` / ``TelemetryBatchAckV1``, + event records, receipts and the :class:`~research.telemetry.ingestion.models.IngestionStore` + protocol. +* :mod:`research.telemetry.ingestion.validation` - per-event schema/context/sensitive scan. +* :mod:`research.telemetry.ingestion.service` - :func:`ingest_batch`, the deterministic + authorize-classify-persist service. +* :mod:`research.telemetry.ingestion.store` - SQLAlchemy persistence and the in-memory + :class:`~research.telemetry.ingestion.store.FakeIngestionStore` test double. + +The core package never imports ``App``, FastAPI, or a session factory. +""" + +from .enums import ( + PERMANENT_REASONS, + RETRYABLE_REASONS, + EventDisposition, + IngestionReasonCode, + is_permanent, + is_retryable, +) +from .errors import IntegrityConflictError, ReceiptConflictError, StoreUnavailable +from .models import ( + BatchReceipt, + EventAck, + IngestionContext, + IngestionIssue, + IngestionStore, + ResearchEventRecord, + TelemetryBatchAckV1, + TelemetryBatchRequestV1, +) +from .service import CapabilityVerifier, compute_event_digest, ingest_batch +from .store import FakeIngestionStore, SqlAlchemyIngestionStore, row_to_record +from .validation import validate_batch_event + +__all__ = [ + "PERMANENT_REASONS", + "RETRYABLE_REASONS", + "BatchReceipt", + "CapabilityVerifier", + "EventAck", + "EventDisposition", + "FakeIngestionStore", + "IngestionContext", + "IngestionIssue", + "IngestionReasonCode", + "IngestionStore", + "IntegrityConflictError", + "ResearchEventRecord", + "ReceiptConflictError", + "SqlAlchemyIngestionStore", + "StoreUnavailable", + "TelemetryBatchAckV1", + "TelemetryBatchRequestV1", + "compute_event_digest", + "ingest_batch", + "is_permanent", + "is_retryable", + "row_to_record", + "validate_batch_event", +] diff --git a/src/research/telemetry/ingestion/agent_tasks.py b/src/research/telemetry/ingestion/agent_tasks.py new file mode 100644 index 0000000..62e6683 --- /dev/null +++ b/src/research/telemetry/ingestion/agent_tasks.py @@ -0,0 +1,165 @@ +"""Ensure acknowledged ACP-proxied runs have one linkable ``agent_task``. + +Dashboards join ``research_event.agent_run_id`` to ``agent_task.external_run_id`` +and scope ownership by ``owner_user_id``. A research-telemetry event carries a +proxy-stamped run id but no task, so the run would be invisible; this module +ensures one task per acknowledged non-null run id, idempotently. Attribution and +ownership come from the stored rows, never the client payload, and the run's +unresolvable profile is stored as non-secret placeholders. +""" + +from __future__ import annotations + +import logging +import uuid +from typing import TYPE_CHECKING, Callable + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError + +from database import crud +from database.research_schemas import ResearchEnrollment, ResearchEvent +from research.participants import identity as identity_store + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + from .models import TelemetryBatchAckV1, TelemetryBatchRequestV1 + +__all__ = ["ensure_agent_tasks_for_ack"] + +logger = logging.getLogger(__name__) + +ACP_TASK_SOURCE = "research-acp" + +#: The plugin mirrors the end of a native agent run as this canonical event; an +#: acknowledged one means the run's task can be finalized. +RUN_COMPLETED_EVENT_TYPE = "agent.run.completed" + +PLACEHOLDER_PROFILE = { + "agent_profile": "acp-external", + "model": "", + "approval_policy": "unknown", + "tools_json": "[]", + "framework_version": "acp-external", +} + + +def _acknowledged_run_events( + payload: TelemetryBatchRequestV1, ack: TelemetryBatchAckV1 +) -> dict[str, list]: + """Group the accepted/duplicate payload events by non-null run id.""" + by_id: dict = {} + for event in payload.events: + by_id.setdefault(event.event_id, event) + grouped: dict[str, list] = {} + seen: set = set() + for entry in (*ack.accepted, *ack.duplicate): + event = by_id.get(entry.event_id) + if entry.event_id in seen or event is None or not event.agent_run_id: + continue + seen.add(entry.event_id) + grouped.setdefault(event.agent_run_id, []).append(event) + return grouped + + +def ensure_agent_tasks_for_ack( + db: Session, + payload: TelemetryBatchRequestV1, + ack: TelemetryBatchAckV1, + *, + before_create: Callable[[uuid.UUID], None] = lambda account_id: None, +) -> list: + """Ensure one linkable ``agent_task`` exists per acknowledged run id. + + ``before_create(account_id)`` runs before a task is created for an account; + the router uses it to lock the account against a concurrent erasure. The + enrollment is then read again, because an erasure that committed meanwhile + has removed it and its run must not get a task. + """ + grouped = _acknowledged_run_events(payload, ack) + if not grouped: + return [] + rows = db.execute( + select( + ResearchEvent.event_id, + ResearchEvent.enrollment_id, + ResearchEvent.research_session_id, + ).where( + ResearchEvent.event_id.in_( + [event.event_id for events in grouped.values() for event in events] + ) + ) + ).all() + stored = {row.event_id: row for row in rows} + created: list = [] + for run_id, events in grouped.items(): + row = next((stored[e.event_id] for e in events if e.event_id in stored), None) + if row is None or row.enrollment_id is None: + continue + enrollment = identity_store.get_enrollment(db, row.enrollment_id) + if enrollment is None: + continue + participant = identity_store.get_participant(db, enrollment.participant_id) + if participant is None or participant.account_id is None: + continue + existing = crud.get_agent_task_by_external_run_id(db, run_id) + if existing is not None: + _finalize_if_completed(db, existing, events) + continue + before_create(participant.account_id) + if db.execute( + select(ResearchEnrollment.enrollment_id).where( + ResearchEnrollment.enrollment_id == enrollment.enrollment_id + ) + ).first() is None: + continue + try: + task = crud.create_agent_task( + db, + **PLACEHOLDER_PROFILE, + source=ACP_TASK_SOURCE, + status="running", + owner_user_id=participant.account_id, + external_run_id=run_id, + study_id=enrollment.study_id, + research_session_id=row.research_session_id, + enrollment_id=enrollment.enrollment_id, + ) + except IntegrityError: + # A concurrent batch won ``external_run_id`` (UNIQUE); roll back and + # re-read the winner instead of failing the surrounding batch. + db.rollback() + crud.get_agent_task_by_external_run_id(db, run_id) + continue + created.append(task) + logger.info( + "linked ACP run %s to agent_task %s", run_id, getattr(task, "task_id", "?") + ) + _finalize_if_completed(db, task, events) + return created + + +def _finalize_if_completed(db: Session, task, events: list) -> None: + """Finalize ``task`` once its run's completion event was acknowledged.""" + if getattr(task, "status", None) not in (None, "pending", "running"): + return + completed = next( + (event for event in events if getattr(event, "event_type", None) == RUN_COMPLETED_EVENT_TYPE), + None, + ) + if completed is None: + return + payload = getattr(completed, "payload", None) or {} + outcome = str(payload.get("outcome") or payload.get("status") or "").lower() + status = ( + "cancelled" if "cancel" in outcome + else "failed" if ("fail" in outcome or "error" in outcome or "crash" in outcome) + else "done" + ) + try: + from agents import lifecycle + + lifecycle.finalize_agent_task(db, task.task_id, status=status) + except Exception as error: # noqa: BLE001 - the ack must never fail on this + logger.warning("could not finalize ACP run task %s: %s", getattr(task, "task_id", "?"), error) diff --git a/src/research/telemetry/ingestion/enums.py b/src/research/telemetry/ingestion/enums.py new file mode 100644 index 0000000..86fbc6b --- /dev/null +++ b/src/research/telemetry/ingestion/enums.py @@ -0,0 +1,110 @@ +"""Closed vocabularies for idempotent telemetry ingestion (Issue 09). + +A disposition is terminal for one event: ``REJECTED`` reasons are permanent +(the event must not be retried) while ``RETRYABLE`` reasons are transient. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = [ + "EventDisposition", + "IngestionReasonCode", + "PERMANENT_REASONS", + "RETRYABLE_REASONS", + "is_permanent", + "is_retryable", +] + + +class EventDisposition(str, Enum): + """Terminal per-event acknowledgement disposition.""" + + ACCEPTED = "ACCEPTED" + DUPLICATE = "DUPLICATE" + REJECTED = "REJECTED" + RETRYABLE = "RETRYABLE" + + +class IngestionReasonCode(str, Enum): + """Stable machine-readable reason for an event disposition.""" + + OK = "OK" + + # Permanent (the event must not be retried). + INVALID_SCHEMA = "INVALID_SCHEMA" + UNKNOWN_SCHEMA_VERSION = "UNKNOWN_SCHEMA_VERSION" + MISSING_PROVENANCE = "MISSING_PROVENANCE" + MISSING_EVENT_ID = "MISSING_EVENT_ID" + CONTEXT_MISMATCH = "CONTEXT_MISMATCH" + # The event_type/source is not a canonical vocabulary member and no + # ``unknown_*`` marker was preserved: it must never be stored as a + # known-looking value (TSCH-01/TSCH-02). + UNKNOWN_EVENT_TYPE = "UNKNOWN_EVENT_TYPE" + UNKNOWN_SOURCE = "UNKNOWN_SOURCE" + SESSION_OUT_OF_SCOPE = "SESSION_OUT_OF_SCOPE" + SESSION_TERMINAL = "SESSION_TERMINAL" + ENROLLMENT_NOT_ACTIVE = "ENROLLMENT_NOT_ACTIVE" + REVOKED = "REVOKED" + INTEGRITY_CONFLICT = "INTEGRITY_CONFLICT" + SENSITIVE_PAYLOAD = "SENSITIVE_PAYLOAD" + PRIVACY_BLOCKED = "PRIVACY_BLOCKED" + EVENT_SEQUENCE_GAP = "EVENT_SEQUENCE_GAP" + RECEIPT_CONFLICT = "RECEIPT_CONFLICT" + DUPLICATE_BATCH = "DUPLICATE_BATCH" + BATCH_TOO_LARGE = "BATCH_TOO_LARGE" + EMPTY_BATCH = "EMPTY_BATCH" + CONTINUITY_REQUIRED = "CONTINUITY_REQUIRED" + + # Transient (safe to retry after re-bootstrap or backoff). + STORE_UNAVAILABLE = "STORE_UNAVAILABLE" + CAPABILITY_INVALID = "CAPABILITY_INVALID" + # Operator kill switch: new batches are not stored while engaged. + KILL_SWITCH_ENGAGED = "KILL_SWITCH_ENGAGED" + + +# Reason codes that permanently reject an event. +PERMANENT_REASONS = frozenset( + { + IngestionReasonCode.INVALID_SCHEMA, + IngestionReasonCode.UNKNOWN_SCHEMA_VERSION, + IngestionReasonCode.MISSING_PROVENANCE, + IngestionReasonCode.MISSING_EVENT_ID, + IngestionReasonCode.CONTEXT_MISMATCH, + IngestionReasonCode.UNKNOWN_EVENT_TYPE, + IngestionReasonCode.UNKNOWN_SOURCE, + IngestionReasonCode.SESSION_OUT_OF_SCOPE, + IngestionReasonCode.SESSION_TERMINAL, + IngestionReasonCode.ENROLLMENT_NOT_ACTIVE, + IngestionReasonCode.REVOKED, + IngestionReasonCode.INTEGRITY_CONFLICT, + IngestionReasonCode.SENSITIVE_PAYLOAD, + IngestionReasonCode.PRIVACY_BLOCKED, + IngestionReasonCode.EVENT_SEQUENCE_GAP, + IngestionReasonCode.RECEIPT_CONFLICT, + IngestionReasonCode.DUPLICATE_BATCH, + IngestionReasonCode.BATCH_TOO_LARGE, + IngestionReasonCode.EMPTY_BATCH, + IngestionReasonCode.CONTINUITY_REQUIRED, + } +) + +# Reason codes that permit a later retry. +RETRYABLE_REASONS = frozenset( + { + IngestionReasonCode.STORE_UNAVAILABLE, + IngestionReasonCode.CAPABILITY_INVALID, + IngestionReasonCode.KILL_SWITCH_ENGAGED, + } +) + + +def is_permanent(reason: IngestionReasonCode) -> bool: + """Whether ``reason`` permanently rejects an event.""" + return reason in PERMANENT_REASONS + + +def is_retryable(reason: IngestionReasonCode) -> bool: + """Whether ``reason`` permits a later retry.""" + return reason in RETRYABLE_REASONS diff --git a/src/research/telemetry/ingestion/errors.py b/src/research/telemetry/ingestion/errors.py new file mode 100644 index 0000000..15afe8a --- /dev/null +++ b/src/research/telemetry/ingestion/errors.py @@ -0,0 +1,48 @@ +"""Typed errors for ingestion store failures.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from uuid import UUID + +__all__ = [ + "IntegrityConflictError", + "ReceiptConflictError", + "StoreUnavailable", +] + + +class StoreUnavailable(RuntimeError): + """Raised when the event store is transiently unavailable. + + The caller must return a retryable acknowledgement, never a false accept. + """ + + +class ReceiptConflictError(RuntimeError): + """Raised when a concurrent commit already stored a receipt for the batch. + + The batch id is unique, so exactly one commit wins. The loser must discard + its own staged writes and return the winner's immutable receipt. + """ + + def __init__(self, batch_id: "UUID") -> None: + super().__init__(f"batch {batch_id} already has a stored receipt") + self.batch_id = batch_id + + +class IntegrityConflictError(RuntimeError): + """Raised when an event id already exists with a different digest.""" + + def __init__( + self, event_id: UUID, stored_digest: str, presented_digest: str + ) -> None: + super().__init__( + f"integrity conflict for event {event_id}: stored digest differs from " + "the presented digest" + ) + self.event_id = event_id + self.stored_digest = stored_digest + self.presented_digest = presented_digest diff --git a/src/research/telemetry/ingestion/models.py b/src/research/telemetry/ingestion/models.py new file mode 100644 index 0000000..9e7b88f --- /dev/null +++ b/src/research/telemetry/ingestion/models.py @@ -0,0 +1,213 @@ +"""Pydantic v2 contracts for idempotent telemetry ingestion (Issue 09). + +``TelemetryBatchRequestV1`` carries already-privacy-filtered canonical events +plus a scoped session capability. ``TelemetryBatchAckV1`` groups event ids by +disposition; an event id never appears in two groups, and a missing +acknowledgement is never treated as accepted. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Any, Optional, Protocol, Sequence +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from research.runtime.bootstrap.models import ( + SessionCapability, # noqa: TC001 - pydantic resolves model annotations at runtime +) +from research.telemetry.models import ( + CanonicalEventV1, # noqa: TC001 - pydantic resolves model annotations at runtime +) + +from .enums import ( + EventDisposition, # noqa: TC001 - pydantic resolves model field annotations at runtime + IngestionReasonCode, # noqa: TC001 - pydantic resolves model field annotations at runtime +) + +_FROZEN = ConfigDict(extra="forbid", frozen=True) +_BASE = ConfigDict(extra="forbid") + + +class TelemetryBatchRequestV1(BaseModel): + """One upload request from the durable local spool.""" + + model_config = _BASE + + batch_id: UUID + protocol_version: str = "1" + telemetry_schema_version: str = "1" + session_capability: SessionCapability + events: list[CanonicalEventV1] = Field(default_factory=list) + client_instance_id: str + + @property + def size(self) -> int: + """Number of events in the batch.""" + return len(self.events) + + +class EventAck(BaseModel): + """Per-event acknowledgement.""" + + model_config = _BASE + + event_id: Optional[UUID] = None + disposition: EventDisposition + reason: Optional[IngestionReasonCode] = None + retry_hint: Optional[int] = None + stored_digest: Optional[str] = None + + +class TelemetryBatchAckV1(BaseModel): + """Durable batch acknowledgement, safe to retry and deterministic per event.""" + + model_config = _BASE + + receipt_id: UUID + batch_id: UUID + server_time: datetime + accepted: list[EventAck] = Field(default_factory=list) + duplicate: list[EventAck] = Field(default_factory=list) + rejected: list[EventAck] = Field(default_factory=list) + retryable: list[EventAck] = Field(default_factory=list) + retry_hint: Optional[int] = None + # Coverage diagnostics (e.g. per-emitter sequence gaps) keyed by event id. + coverage_diagnostics: dict[str, str] = Field(default_factory=dict) + + @model_validator(mode="after") + def _unique_event_ids(self) -> TelemetryBatchAckV1: + seen: set[UUID] = set() + for group in (self.accepted, self.duplicate, self.rejected, self.retryable): + for ack in group: + if ack.event_id is None: + continue + if ack.event_id in seen: + raise ValueError( + f"event {ack.event_id} appears in more than one ack group" + ) + seen.add(ack.event_id) + return self + + def acknowledged_ids(self) -> list[UUID]: + """Ids the spool may delete: accepted + duplicate.""" + return [ack.event_id for ack in self.accepted + self.duplicate if ack.event_id] + + @property + def is_empty(self) -> bool: + """Whether the acknowledgement carries no per-event dispositions.""" + return not (self.accepted or self.duplicate or self.rejected or self.retryable) + + +class IngestionContext(BaseModel): + """Authorized, resolved context every accepted event is bound to. + + Every identity field is derived from the authorized enrollment/session, never + from the client event payload. + """ + + model_config = _FROZEN + + study_id: UUID + enrollment_id: UUID + research_session_id: UUID + revocation_epoch: int = 0 + + +class IngestionIssue(BaseModel): + """One typed validation finding for a batch event.""" + + model_config = _BASE + + code: IngestionReasonCode + field: str = "" + message: str = "" + permanent: bool = True + + +class ResearchEventRecord(BaseModel): + """One immutable accepted canonical event fact. + + ``envelope`` is the complete validated ``CanonicalEventV1`` document and is + the event authority: it preserves correlations, monotonic time, lifecycle, + unknown-value fields, provenance, coverage and the privacy summary. The + remaining fields are the searchable scalar columns derived from that same + envelope in the same transaction (identity/ordering for indexes and joins); + they are never a second, independently writable copy of the content. + Ingestion metadata (``digest``, ``accepted_at``, ``retention_state``) lives + outside the envelope. + """ + + model_config = _FROZEN + + event_id: UUID + schema_version: str + event_type: str + source: str + study_id: Optional[UUID] = None + enrollment_id: Optional[UUID] = None + research_session_id: Optional[UUID] = None + agent_run_id: Optional[str] = None + emitter_id: str + emitter_sequence: int + occurred_at: datetime + # The complete validated canonical envelope (the authority). + envelope: dict[str, Any] = Field(default_factory=dict) + digest: str + accepted_at: datetime + retention_state: str = "RETAINED" + + +class BatchReceipt(BaseModel): + """An immutable, retry-safe batch acknowledgement record.""" + + model_config = _FROZEN + + receipt_id: UUID + batch_id: UUID + enrollment_id: Optional[UUID] = None + research_session_id: Optional[UUID] = None + accepted_at: datetime + ack: TelemetryBatchAckV1 + + +class IngestionStore(Protocol): + """The persistence surface the ingestion service depends on. + + The service owns the transaction boundary: ``insert_events`` and + ``record_batch_receipt`` stage their writes, and ``commit``/``rollback`` + decide whether the whole batch (accepted events + immutable receipt) becomes + durable. A batch therefore commits in full or not at all. + """ + + def get_stored_digests( + self, event_ids: Sequence[UUID] + ) -> dict[UUID, str]: # pragma: no cover - structural protocol + ... + + def insert_events( + self, records: Sequence[ResearchEventRecord] + ) -> list[UUID]: # pragma: no cover - structural protocol + ... + + def record_batch_receipt( + self, receipt: BatchReceipt + ) -> None: # pragma: no cover - structural protocol + ... + + def get_receipt( + self, batch_id: UUID + ) -> Optional[BatchReceipt]: # pragma: no cover - structural protocol + ... + + def get_emitter_cursor( + self, emitter_id: str, research_session_id: Optional[UUID] + ) -> Optional[int]: # pragma: no cover - structural protocol + ... + + def commit(self) -> None: # pragma: no cover - structural protocol + ... + + def rollback(self) -> None: # pragma: no cover - structural protocol + ... diff --git a/src/research/telemetry/ingestion/service.py b/src/research/telemetry/ingestion/service.py new file mode 100644 index 0000000..ba3fdd0 --- /dev/null +++ b/src/research/telemetry/ingestion/service.py @@ -0,0 +1,963 @@ +"""Idempotent batch ingestion service (Issue 09). + +Authorization happens inside the same transaction that persists the batch: the +scoped session capability is verified (signature, audience, scope, expiry, +revocation epoch), the enrollment and research session rows are locked +(``SELECT ... FOR UPDATE``) and validated, the events are classified, and the +accepted events plus the immutable batch receipt are committed together. The +ACK is returned only after commit, so a mid-batch database error rolls the whole +batch back and produces a retryable result — never a partial accept. + +Per event, classification is deterministic given store state: + +* same ``event_id`` + same digest already stored -> ``DUPLICATE`` (acknowledged, + never a second fact); +* same ``event_id`` + different digest -> ``REJECTED`` integrity conflict, never + overwritten; +* invalid/unsafe/out-of-scope/blocked -> ``REJECTED`` with a permanent typed + reason; +* transient store failure -> ``RETRYABLE`` with a hint and no false ACK; +* otherwise -> ``ACCEPTED``. + +The returned acknowledgement is immutable and retry-safe: re-POSTing the same +``batch_id`` returns the stored receipt unchanged, and a concurrent duplicate +loses the receipt race but returns the winner's receipt (never a second one). +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Callable, Optional, Protocol, Sequence + +from research.canonical import canonical_hash +from research.participants.enums import EnrollmentStatus +from research.runtime.bootstrap.models import CapabilityReasonCode +from research.runtime.sessions.enums import SessionState +from research.telemetry.privacy import PrivacyPolicy, filter_event + +from .enums import EventDisposition, IngestionReasonCode +from .errors import IntegrityConflictError, ReceiptConflictError, StoreUnavailable +from .models import ( + BatchReceipt, + EventAck, + IngestionContext, + IngestionStore, + ResearchEventRecord, + TelemetryBatchAckV1, + TelemetryBatchRequestV1, +) +from .validation import validate_batch_event + +if TYPE_CHECKING: + from research.participants.models import Enrollment + from research.runtime.bootstrap.models import CapabilityVerification, SessionCapability + from research.runtime.sessions.models import ResearchSessionV1 + from research.telemetry.models import CanonicalEventV1 + +__all__ = ["CapabilityVerifier", "compute_event_digest", "ingest_batch"] + +DEFAULT_RETRY_HINT_SECONDS = 30 +DEFAULT_MAX_EVENTS = 500 +# How long after a session was closed (ENDED, not REVOKED) its late events are +# still stored (C-02): the last poll window, the closing events and the tail +# collected offline before the client learned the session had ended. +DEFAULT_LATE_EVENT_GRACE_SECONDS = 900 + + +class CapabilityVerifier(Protocol): + """Verifies a session capability against its resolved subject. + + The resolved ``(study, enrollment, session)`` are passed so the verifier + can reject a cryptographically valid capability that was issued for a + different subject. Verifiers may accept them as optional keyword arguments + (the default ``None`` skips the subject comparison). + """ + + def __call__( + self, + capability: SessionCapability, + *, + now: datetime, + current_revocation_epoch: Optional[int], + expected_enrollment_id: Optional[uuid.UUID] = None, + expected_research_session_id: Optional[uuid.UUID] = None, + expected_study_id: Optional[uuid.UUID] = None, + ) -> CapabilityVerification: # pragma: no cover - structural protocol + ... + + +def _now(now: Optional[datetime]) -> datetime: + return now or datetime.now(timezone.utc) + + +def compute_event_digest(event: CanonicalEventV1) -> str: + """Content digest of a canonical event (stable, order-independent).""" + return canonical_hash(event.model_dump(mode="json")) + + +def _record_from_event( + event: CanonicalEventV1, + context: IngestionContext, + digest: str, + accepted_at: datetime, +) -> ResearchEventRecord: + """Build the stored record from the *authorized* context. + + ``study_id``/``enrollment_id``/``research_session_id`` are + always taken from the authorized enrollment/session, never from the client + event payload. A payload field that disagrees with the context is rejected + by :func:`validate_batch_event` before this runs. + """ + return ResearchEventRecord( + event_id=event.event_id, + schema_version=event.schema_version, + event_type=event.event_type, + source=event.source, + study_id=context.study_id, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + agent_run_id=event.agent_run_id, + emitter_id=event.emitter_id, + emitter_sequence=event.emitter_sequence, + occurred_at=event.occurred_at, + # The complete validated envelope is the stored authority; the scalar + # fields above are searchable projections derived from it. + envelope=event.model_dump(mode="json"), + digest=digest, + accepted_at=accepted_at, + retention_state="RETAINED", + ) + + +def _ack( + batch_id: uuid.UUID, + server_time: datetime, + *, + accepted: Sequence[EventAck] = (), + duplicate: Sequence[EventAck] = (), + rejected: Sequence[EventAck] = (), + retryable: Sequence[EventAck] = (), + retry_hint: Optional[int] = None, + diagnostics: Optional[dict[str, str]] = None, +) -> TelemetryBatchAckV1: + return TelemetryBatchAckV1( + receipt_id=uuid.uuid4(), + batch_id=batch_id, + server_time=server_time, + accepted=list(accepted), + duplicate=list(duplicate), + rejected=list(rejected), + retryable=list(retryable), + retry_hint=retry_hint, + coverage_diagnostics=dict(diagnostics or {}), + ) + + +def _retryable_ack(ack: TelemetryBatchAckV1) -> TelemetryBatchAckV1: + """Re-target every accepted event as retryable (no false ACK).""" + retryable = [ + EventAck( + event_id=event_ack.event_id, + disposition=EventDisposition.RETRYABLE, + reason=IngestionReasonCode.STORE_UNAVAILABLE, + ) + for event_ack in ack.accepted + ] + return TelemetryBatchAckV1( + receipt_id=uuid.uuid4(), + batch_id=ack.batch_id, + server_time=ack.server_time, + duplicate=list(ack.duplicate), + rejected=list(ack.rejected), + retryable=list(ack.retryable) + retryable, + retry_hint=ack.retry_hint, + coverage_diagnostics=dict(ack.coverage_diagnostics), + ) + + +def _finalize( + store: IngestionStore, + batch_id: uuid.UUID, + ack: TelemetryBatchAckV1, + *, + records: Sequence[ResearchEventRecord] = (), + enrollment_id: Optional[uuid.UUID], + research_session_id: Optional[uuid.UUID], + retry_hint: Optional[int] = None, +) -> TelemetryBatchAckV1: + """Persist the accepted events and the immutable receipt in one transaction. + + A retryable batch is not cached (a retry must be retryable): any row locks + acquired for the batch are released with a rollback. A terminal decision + stages the events and receipt, commits once, and only then returns the ACK. + + An :class:`IntegrityConflictError` identifies a single poison event; it is + terminally rejected and the remaining records are persisted in a fresh + transaction, so one bad event cannot make its siblings retryable forever. + """ + if ack.retryable: + store.rollback() + return ack + + receipt = BatchReceipt( + receipt_id=ack.receipt_id, + batch_id=batch_id, + enrollment_id=enrollment_id, + research_session_id=research_session_id, + accepted_at=ack.server_time, + ack=ack, + ) + try: + if records: + store.insert_events(records) + store.record_batch_receipt(receipt) + store.commit() + except ReceiptConflictError: + # A concurrent submission already committed a receipt for this batch: + # never a second one. Return the winner's immutable receipt. + store.rollback() + existing = store.get_receipt(batch_id) + return existing.ack if existing is not None else _retryable_ack(ack) + except IntegrityConflictError as conflict: + store.rollback() + # A single poison event (its event-id/digest identity is already taken + # by a different fact) must not wedge the whole batch: terminally + # reject only that event and persist the survivors. + # Each recursion removes exactly one record, so this is bounded by the + # batch size and always terminates. + survivors = [ + record for record in records if record.event_id != conflict.event_id + ] + if len(survivors) == len(records): + # The conflict is not one of the staged records (for example a + # concurrent commit we cannot attribute): never loop, and never + # falsely acknowledge; fall back to a retryable batch. + return _retryable_ack(ack) + rejected = list(ack.rejected) + [ + EventAck( + event_id=conflict.event_id, + disposition=EventDisposition.REJECTED, + reason=IngestionReasonCode.INTEGRITY_CONFLICT, + stored_digest=conflict.stored_digest, + ) + ] + accepted_survivors = [ + event_ack + for event_ack in ack.accepted + if event_ack.event_id != conflict.event_id + ] + conflict_ack = TelemetryBatchAckV1( + receipt_id=uuid.uuid4(), + batch_id=ack.batch_id, + server_time=ack.server_time, + accepted=accepted_survivors, + duplicate=list(ack.duplicate), + rejected=rejected, + retryable=list(ack.retryable), + retry_hint=retry_hint, + coverage_diagnostics=dict(ack.coverage_diagnostics), + ) + return _finalize( + store, + batch_id, + conflict_ack, + records=survivors, + enrollment_id=enrollment_id, + research_session_id=research_session_id, + retry_hint=retry_hint, + ) + except StoreUnavailable: + store.rollback() + return _retryable_ack(ack) + + # The unique ``batch_id`` guarantees one owner. If a concurrent submission + # won, return its stored receipt rather than this (unstored) ACK. + stored = store.get_receipt(batch_id) + if stored is not None and stored.receipt_id != ack.receipt_id: + return stored.ack + return ack + + +def _reject_all( + store: IngestionStore, + batch_id: uuid.UUID, + events: Sequence[CanonicalEventV1], + server_time: datetime, + reason: IngestionReasonCode, + *, + disposition: EventDisposition, + retry_hint: Optional[int] = None, + enrollment_id: Optional[uuid.UUID] = None, + research_session_id: Optional[uuid.UUID] = None, + max_acks: Optional[int] = None, + persist_receipt: bool = True, +) -> TelemetryBatchAckV1: + """Reject every event in the batch with one typed reason. + + ``persist_receipt=False`` is used for pre-authorization rejections (an + unknown/out-of-scope subject, an oversized batch): those must not write a + durable receipt, otherwise an unauthenticated caller could grow the receipt + table one row per invented batch id. ``max_acks`` bounds the per-event ack + list so an oversized batch cannot build an unbounded response. + """ + bounded = list(events[:max_acks]) if max_acks is not None else list(events) + group = [ + EventAck(event_id=event.event_id, disposition=disposition, reason=reason) + for event in bounded + ] + if disposition == EventDisposition.RETRYABLE: + ack = _ack(batch_id, server_time, retryable=group, retry_hint=retry_hint) + else: + ack = _ack(batch_id, server_time, rejected=group) + if not persist_receipt: + return ack + return _finalize( + store, + batch_id, + ack, + enrollment_id=enrollment_id, + research_session_id=research_session_id, + retry_hint=retry_hint, + ) + + +def _anchor_context( + request: TelemetryBatchRequestV1, + enrollment_resolver: Callable[[uuid.UUID], Optional[Enrollment]], + session_resolver: Callable[[uuid.UUID], Optional[ResearchSessionV1]], +) -> Optional[tuple[IngestionContext, Enrollment, ResearchSessionV1]]: + """Resolve (and lock) the authorized subject of the batch. + + The resolvers are expected to lock the enrollment and session rows + (``SELECT ... FOR UPDATE``); this serializes concurrent batches for the same + subject, which is what makes batch idempotency concurrency-safe. + """ + for event in request.events: + if event.research_session_id is None or event.enrollment_id is None: + continue + session = session_resolver(event.research_session_id) + if session is None: + continue + enrollment = enrollment_resolver(event.enrollment_id) + if enrollment is None: + continue + if session.enrollment_id != enrollment.enrollment_id: + continue + if enrollment.study_id != session.study_id: + continue + context = IngestionContext( + study_id=enrollment.study_id, + enrollment_id=enrollment.enrollment_id, + research_session_id=session.research_session_id, + revocation_epoch=enrollment.revocation_epoch, + ) + return context, enrollment, session + return None + + +def _is_terminal(session: ResearchSessionV1) -> bool: + state = session.state + return state.is_terminal if hasattr(state, "is_terminal") else False + + +def _late_acceptance_deadline( + session: ResearchSessionV1, grace_seconds: int +) -> Optional[datetime]: + """The last ``occurred_at`` still accepted for an ENDED session, or ``None``. + + Only an ordinary end (idle timeout, explicit completion, IDE closed, ...) + gets a grace window: a REVOKED session (withdrawal, consent pause, study + stop cascade) stays fully terminal, and a session with no ``closed_at`` + cannot anchor a window. + """ + if getattr(session, "state", None) != SessionState.ENDED: + return None + closed_at = getattr(session, "closed_at", None) + if closed_at is None or grace_seconds < 0: + return None + if closed_at.tzinfo is None: + closed_at = closed_at.replace(tzinfo=timezone.utc) + return closed_at + timedelta(seconds=grace_seconds) + + +def _occurred_within(event: CanonicalEventV1, deadline: datetime) -> bool: + occurred_at = event.occurred_at + if occurred_at.tzinfo is None: + occurred_at = occurred_at.replace(tzinfo=timezone.utc) + return occurred_at <= deadline + + +def _receipt_denied_ack( + request: TelemetryBatchRequestV1, server_time: datetime, retry_hint: int +) -> TelemetryBatchAckV1: + """A refusal that leaks no receipt content and never writes a new receipt.""" + return _ack( + request.batch_id, + server_time, + retryable=[ + EventAck( + event_id=event.event_id, + disposition=EventDisposition.RETRYABLE, + reason=IngestionReasonCode.CAPABILITY_INVALID, + ) + for event in request.events + ], + retry_hint=retry_hint, + ) + + +def _authorized_receipt( + request: TelemetryBatchRequestV1, + receipt: BatchReceipt, + *, + receipt_capability_verifier: CapabilityVerifier, + server_time: datetime, + retry_hint: int, + session_resolver: Optional[Callable[[uuid.UUID], Optional[ResearchSessionV1]]] = None, +) -> TelemetryBatchAckV1: + """Return a stored receipt only to the capability's own subject (ISSUE-08). + + The batch id alone is never sufficient: the signed capability must cover + the receipt's enrollment/session subject. Receipts persisted before subject + binding existed (no enrollment/session) are refused rather than echoed. + + A receipt for a terminal session's batch (an ENDED session's tail accepted + under C-02, or a REVOKED session's permanent rejections) was issued to the + enrollment's *next* session capability, so a lost-ACK retry of that batch + arrives with a capability naming another session. The session binding is + relaxed to the enrollment for exactly that case (the receipt's session is + terminal); otherwise the replay would be refused forever and, because the + client retries the oldest batch first, every later upload would stall + behind it. + """ + if receipt.enrollment_id is None and receipt.research_session_id is None: + return _receipt_denied_ack(request, server_time, retry_hint) + expected_session = receipt.research_session_id + if ( + expected_session is not None + and request.session_capability.research_session_id != expected_session + and session_resolver is not None + ): + receipt_session = session_resolver(expected_session) + if receipt_session is not None and _is_terminal(receipt_session): + expected_session = None + verification = receipt_capability_verifier( + request.session_capability, + now=server_time, + current_revocation_epoch=None, + expected_enrollment_id=receipt.enrollment_id, + expected_research_session_id=expected_session, + ) + if not verification.ok: + return _receipt_denied_ack(request, server_time, retry_hint) + return receipt.ack + + +def ingest_batch( + request: TelemetryBatchRequestV1, + *, + capability_verifier: CapabilityVerifier, + enrollment_resolver: Callable[[uuid.UUID], Optional[Enrollment]], + session_resolver: Callable[[uuid.UUID], Optional[ResearchSessionV1]], + store: IngestionStore, + now: Optional[datetime] = None, + max_events: int = DEFAULT_MAX_EVENTS, + continuity_required: bool = False, + retry_hint: int = DEFAULT_RETRY_HINT_SECONDS, + kill_switch_check: Optional[Callable[[], bool]] = None, + privacy_policy_resolver: Optional[Callable[[Enrollment], PrivacyPolicy]] = None, + receipt_capability_verifier: Optional[CapabilityVerifier] = None, + late_event_grace_seconds: int = DEFAULT_LATE_EVENT_GRACE_SECONDS, +) -> TelemetryBatchAckV1: + """Ingest one batch and return a durable, deterministic acknowledgement. + + ``late_event_grace_seconds`` (C-02): events anchored on an ``ENDED`` session + are still stored when the capability verifies for that session's + enrollment and study (the session binding is relaxed, so the client's + *next* session capability may deliver the previous session's tail) and + each event occurred no later than ``closed_at + grace``; later events are + rejected ``SESSION_TERMINAL``. ``REVOKED`` sessions reject everything. + """ + server_time = _now(now) + + # Retry of an already-received batch returns the immutable receipt, but only + # after the capability proves the caller owns its subject (ISSUE-08). + existing_receipt = store.get_receipt(request.batch_id) + if existing_receipt is not None: + return _authorized_receipt( + request, + existing_receipt, + receipt_capability_verifier=( + receipt_capability_verifier or capability_verifier + ), + server_time=server_time, + retry_hint=retry_hint, + session_resolver=session_resolver, + ) + + if len(request.events) > max_events: + return _reject_all( + store, + request.batch_id, + request.events, + server_time, + IngestionReasonCode.BATCH_TOO_LARGE, + disposition=EventDisposition.REJECTED, + retry_hint=retry_hint, + max_acks=max_events, + persist_receipt=False, + ) + + # Operator kill switch (Issue 13): accepted batches are not stored while + # engaged. The check is injected so ingestion keeps no operations dependency. + if kill_switch_check is not None and kill_switch_check(): + kill_switch_group = [ + EventAck( + event_id=event.event_id, + disposition=EventDisposition.RETRYABLE, + reason=IngestionReasonCode.KILL_SWITCH_ENGAGED, + ) + for event in request.events + ] + return _ack( + request.batch_id, server_time, retryable=kill_switch_group, retry_hint=retry_hint + ) + + if not request.events: + # An empty batch has no subject to anchor or authorize: refuse it with a + # typed reason and never persist a durable receipt for an + # unauthenticated caller (ISSUE-08). + return _ack( + request.batch_id, + server_time, + retry_hint=retry_hint, + diagnostics={"batch": IngestionReasonCode.EMPTY_BATCH.value}, + ) + + anchored = _anchor_context(request, enrollment_resolver, session_resolver) + if anchored is None: + return _reject_all( + store, + request.batch_id, + request.events, + server_time, + IngestionReasonCode.SESSION_OUT_OF_SCOPE, + disposition=EventDisposition.REJECTED, + retry_hint=retry_hint, + persist_receipt=False, + ) + context, enrollment, session = anchored + + # Re-check after acquiring the subject locks: a concurrent submission of the + # same batch_id has now either committed (return its receipt) or is blocked + # behind the same lock. + locked_receipt = store.get_receipt(request.batch_id) + if locked_receipt is not None: + store.rollback() + return _authorized_receipt( + request, + locked_receipt, + receipt_capability_verifier=( + receipt_capability_verifier or capability_verifier + ), + server_time=server_time, + retry_hint=retry_hint, + session_resolver=session_resolver, + ) + + late_deadline = _late_acceptance_deadline(session, late_event_grace_seconds) + verification = capability_verifier( + request.session_capability, + now=server_time, + current_revocation_epoch=enrollment.revocation_epoch, + expected_enrollment_id=context.enrollment_id, + # A terminal session's events are delivered under the enrollment's + # next session capability: bind to enrollment + study, not to the + # ended/revoked session. For an ENDED session that admits the tail + # within the grace window; for a REVOKED one it makes the refusal the + # permanent SESSION_TERMINAL below instead of a retryable capability + # mismatch the client would retry forever, starving every later batch. + expected_research_session_id=( + None if _is_terminal(session) else context.research_session_id + ), + expected_study_id=context.study_id, + ) + if not verification.ok: + if verification.reason == CapabilityReasonCode.REVOKED: + return _reject_all( + store, + request.batch_id, + request.events, + server_time, + IngestionReasonCode.REVOKED, + disposition=EventDisposition.REJECTED, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + retry_hint=retry_hint, + ) + return _reject_all( + store, + request.batch_id, + request.events, + server_time, + IngestionReasonCode.CAPABILITY_INVALID, + disposition=EventDisposition.RETRYABLE, + retry_hint=retry_hint, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + ) + + if enrollment.status != EnrollmentStatus.ACTIVE: + return _reject_all( + store, + request.batch_id, + request.events, + server_time, + IngestionReasonCode.ENROLLMENT_NOT_ACTIVE, + disposition=EventDisposition.REJECTED, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + retry_hint=retry_hint, + ) + + events: Sequence[CanonicalEventV1] = request.events + late_rejected: list[EventAck] = [] + if _is_terminal(session): + if late_deadline is None: + # REVOKED (or an ENDED session with no close time): no later event + # is ever stored. + return _reject_all( + store, + request.batch_id, + request.events, + server_time, + IngestionReasonCode.SESSION_TERMINAL, + disposition=EventDisposition.REJECTED, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + retry_hint=retry_hint, + ) + # ENDED: the tail up to the grace deadline is an ordinary fact of that + # session; anything later is terminally rejected per event. + in_grace: list[CanonicalEventV1] = [] + for event in request.events: + if _occurred_within(event, late_deadline): + in_grace.append(event) + else: + late_rejected.append( + EventAck( + event_id=event.event_id, + disposition=EventDisposition.REJECTED, + reason=IngestionReasonCode.SESSION_TERMINAL, + ) + ) + events = in_grace + + return _ingest_authorized_events( + store, + batch_id=request.batch_id, + events=events, + context=context, + telemetry_schema_version=request.telemetry_schema_version, + server_time=server_time, + continuity_required=continuity_required, + retry_hint=retry_hint, + privacy_policy=privacy_policy_resolver(enrollment) if privacy_policy_resolver else PrivacyPolicy.default(), + pre_rejected=late_rejected, + ) + + +def _ingest_authorized_events( + store: IngestionStore, + *, + batch_id: uuid.UUID, + events: Sequence[CanonicalEventV1], + context: IngestionContext, + telemetry_schema_version: str, + server_time: datetime, + continuity_required: bool, + retry_hint: int, + privacy_policy: Optional[PrivacyPolicy] = None, + pre_rejected: Sequence[EventAck] = (), +) -> TelemetryBatchAckV1: + """Persist already-authorized events with the HTTP route's exact semantics. + + ``pre_rejected`` carries per-event rejections the caller already decided + (an ENDED session's events past the late-acceptance deadline); they join + the receipt so the client discards exactly those ids. + + Shared by the participant upload route and the server-authorized internal + entry point: identical per-event validation, digest/uniqueness handling, + sequence continuity, receipt and commit-before-ACK. The caller has already + verified the capability (route) or derived the context server-side + (internal adapter) and applied the terminal/enrollment rules. + """ + accepted: list[EventAck] = [] + duplicate: list[EventAck] = [] + rejected: list[EventAck] = list(pre_rejected) + retryable: list[EventAck] = [] + diagnostics: dict[str, str] = {} + candidates: list[tuple[CanonicalEventV1, str]] = [] + + # An event id may appear more than once in one request (a producer retry + # folded into a batch). The first occurrence wins; the repeats are not a + # second fact and are reported as DUPLICATE below, so an id is acknowledged + # exactly once and never appears in two ack groups (TI-02). + unique_events: list[CanonicalEventV1] = [] + repeated_ids: set[uuid.UUID] = set() + seen_ids: set[uuid.UUID] = set() + for event in events: + if event.event_id in seen_ids: + repeated_ids.add(event.event_id) + continue + seen_ids.add(event.event_id) + unique_events.append(event) + + try: + stored = store.get_stored_digests([event.event_id for event in unique_events]) + except StoreUnavailable: + return _reject_all( + store, + batch_id, + unique_events, + server_time, + IngestionReasonCode.STORE_UNAVAILABLE, + disposition=EventDisposition.RETRYABLE, + retry_hint=retry_hint, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + ) + + for event in unique_events: + issues = validate_batch_event( + event, + context, + telemetry_schema_version=telemetry_schema_version, + ) + if issues: + code = issues[0].code + rejected.append( + EventAck( + event_id=event.event_id, + disposition=EventDisposition.REJECTED, + reason=code, + ) + ) + continue + + if privacy_policy is not None: + filtered = filter_event(event, privacy_policy) + if filtered.summary.blocked or filtered.event.payload != event.payload: + # Do not rewrite canonical bytes: that would invalidate the + # producer's digest and turn a retry into an integrity conflict. + rejected.append(EventAck( + event_id=event.event_id, + disposition=EventDisposition.REJECTED, + reason=IngestionReasonCode.PRIVACY_BLOCKED, + )) + continue + + digest = compute_event_digest(event) + existing_digest = stored.get(event.event_id) + if existing_digest is not None: + if existing_digest == digest: + duplicate.append( + EventAck( + event_id=event.event_id, + disposition=EventDisposition.DUPLICATE, + stored_digest=existing_digest, + ) + ) + else: + rejected.append( + EventAck( + event_id=event.event_id, + disposition=EventDisposition.REJECTED, + reason=IngestionReasonCode.INTEGRITY_CONFLICT, + stored_digest=existing_digest, + ) + ) + continue + + candidates.append((event, digest)) + + # Sequence continuity: a gap is accepted with a coverage diagnostic unless + # the protocol declares continuity required. + sequenced: list[tuple[CanonicalEventV1, str]] = [] + in_batch_last: dict[tuple[str, Optional[uuid.UUID]], int] = {} + for event, digest in candidates: + key = (event.emitter_id, event.research_session_id or context.research_session_id) + cursor = store.get_emitter_cursor(key[0], key[1]) + last = max(cursor or 0, in_batch_last.get(key, 0)) + expected = last + 1 + if event.emitter_sequence != expected: + if continuity_required: + rejected.append( + EventAck( + event_id=event.event_id, + disposition=EventDisposition.REJECTED, + reason=IngestionReasonCode.CONTINUITY_REQUIRED, + ) + ) + in_batch_last[key] = max(last, event.emitter_sequence) + continue + diagnostics[str(event.event_id)] = ( + f"expected emitter sequence {expected}, received " + f"{event.emitter_sequence}" + ) + accepted.append( + EventAck( + event_id=event.event_id, + disposition=EventDisposition.ACCEPTED, + reason=IngestionReasonCode.EVENT_SEQUENCE_GAP, + stored_digest=digest, + ) + ) + else: + accepted.append( + EventAck( + event_id=event.event_id, + disposition=EventDisposition.ACCEPTED, + stored_digest=digest, + ) + ) + in_batch_last[key] = max(last, event.emitter_sequence) + sequenced.append((event, digest)) + + # A repeated id's single ack is a DUPLICATE: the first occurrence is the + # stored fact, the repeats add no second ack (TI-02). + if repeated_ids: + moved = [ack for ack in accepted if ack.event_id in repeated_ids] + accepted = [ack for ack in accepted if ack.event_id not in repeated_ids] + duplicate.extend( + ack.model_copy( + update={ + "disposition": EventDisposition.DUPLICATE, + "reason": None, + } + ) + for ack in moved + ) + + records = [ + _record_from_event(event, context, digest, server_time) + for event, digest in sequenced + ] + ack = _ack( + batch_id, + server_time, + accepted=accepted, + duplicate=duplicate, + rejected=rejected, + retryable=retryable, + retry_hint=retry_hint, + diagnostics=diagnostics, + ) + return _finalize( + store, + batch_id, + ack, + records=records, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + retry_hint=retry_hint, + ) + + +def ingest_events_for_context( + *, + context: IngestionContext, + enrollment: Enrollment, + session: ResearchSessionV1, + events: Sequence[CanonicalEventV1], + store: IngestionStore, + now: Optional[datetime] = None, + kill_switch_check: Optional[Callable[[], bool]] = None, + continuity_required: bool = False, + telemetry_schema_version: str = "1", + max_events: int = DEFAULT_MAX_EVENTS, + retry_hint: int = DEFAULT_RETRY_HINT_SECONDS, + privacy_policy: Optional[PrivacyPolicy] = None, +) -> TelemetryBatchAckV1: + """Server-authorized internal ingestion entry point (no HTTP hop). + + A narrow adapter calls this directly with a context it has already derived + from the authorized account + frozen assignment (an explicit research + binding), so legacy producers stop writing a parallel store. It applies the + same kill-switch, privacy-policy, enrollment-ACTIVE and terminal-session + rules as the participant upload route, then shares the route's validation, + uniqueness, receipt and commit-before-ACK semantics. The + context/assignment is trusted server-side state, never a caller-supplied + body. + """ + server_time = _now(now) + batch_id = uuid.uuid4() + + if len(events) > max_events: + return _reject_all( + store, + batch_id, + events, + server_time, + IngestionReasonCode.BATCH_TOO_LARGE, + disposition=EventDisposition.REJECTED, + retry_hint=retry_hint, + ) + + if kill_switch_check is not None and kill_switch_check(): + group = [ + EventAck( + event_id=event.event_id, + disposition=EventDisposition.RETRYABLE, + reason=IngestionReasonCode.KILL_SWITCH_ENGAGED, + ) + for event in events + ] + return _ack(batch_id, server_time, retryable=group, retry_hint=retry_hint) + + if not events: + return _ack( + batch_id, + server_time, + retry_hint=retry_hint, + diagnostics={"batch": IngestionReasonCode.EMPTY_BATCH.value}, + ) + + if enrollment.status != EnrollmentStatus.ACTIVE: + return _reject_all( + store, + batch_id, + events, + server_time, + IngestionReasonCode.ENROLLMENT_NOT_ACTIVE, + disposition=EventDisposition.REJECTED, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + retry_hint=retry_hint, + ) + + if _is_terminal(session): + return _reject_all( + store, + batch_id, + events, + server_time, + IngestionReasonCode.SESSION_TERMINAL, + disposition=EventDisposition.REJECTED, + enrollment_id=context.enrollment_id, + research_session_id=context.research_session_id, + retry_hint=retry_hint, + ) + + return _ingest_authorized_events( + store, + batch_id=batch_id, + events=events, + context=context, + telemetry_schema_version=telemetry_schema_version, + server_time=server_time, + continuity_required=continuity_required, + retry_hint=retry_hint, + privacy_policy=privacy_policy, + ) diff --git a/src/research/telemetry/ingestion/store.py b/src/research/telemetry/ingestion/store.py new file mode 100644 index 0000000..39c7ab9 --- /dev/null +++ b/src/research/telemetry/ingestion/store.py @@ -0,0 +1,324 @@ +"""Persistence helpers and an in-memory test double for ingestion. + +The SQLAlchemy store takes a caller-supplied ``Session`` and stages its writes: +``insert_events`` and ``record_batch_receipt`` never commit, so the ingestion +service can persist the accepted events and the immutable batch receipt in one +transaction and return the ACK only after ``commit``. ``FakeIngestionStore`` is +a pure in-memory double that mirrors the same commit/rollback semantics (and +enforces unique event ids/digest conflicts), so atomicity is machine-testable +without PostgreSQL. +""" + +from __future__ import annotations + +import logging +import uuid +from typing import TYPE_CHECKING, Optional, Sequence + +from sqlalchemy import func, select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.exc import IntegrityError, SQLAlchemyError + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + +from database.research_schemas import ( + ResearchEvent, + TelemetryBatchReceipt, +) + +from .errors import IntegrityConflictError, ReceiptConflictError, StoreUnavailable +from .models import BatchReceipt, IngestionStore, ResearchEventRecord + +logger = logging.getLogger(__name__) + +__all__ = [ + "FakeIngestionStore", + "IngestionStore", + "SqlAlchemyIngestionStore", + "get_receipt_by_id", + "row_to_record", +] + + +class FakeIngestionStore: + """In-memory ingestion store for deterministic tests. + + Writes are staged until :meth:`commit`, so a failed commit (or a mid-batch + error) leaves the committed state untouched. ``fail_inserts`` simulates a + transient failure while staging events, ``fail_receipts`` while staging the + receipt and ``fail_commit`` at commit time. + """ + + def __init__(self) -> None: + self._events: dict[uuid.UUID, ResearchEventRecord] = {} + self._receipts: dict[uuid.UUID, BatchReceipt] = {} + self._staged_events: dict[uuid.UUID, ResearchEventRecord] = {} + self._staged_receipts: dict[uuid.UUID, BatchReceipt] = {} + self.fail_inserts = False + self.fail_receipts = False + self.fail_commit = False + + # -- events ----------------------------------------------------------- + + def get_stored_digests( + self, event_ids: Sequence[uuid.UUID] + ) -> dict[uuid.UUID, str]: + """Return ``{event_id: digest}`` for event ids already committed.""" + return { + event_id: self._events[event_id].digest + for event_id in event_ids + if event_id in self._events + } + + def insert_events(self, records: Sequence[ResearchEventRecord]) -> list[uuid.UUID]: + """Stage records, enforcing unique ids and digests. + + A same-id/same-digest insert is a no-op (idempotent); a same-id/different + digest raises :class:`IntegrityConflictError`. Like the database, a + reused ``(session, emitter, sequence)`` key is NOT a conflict (C-01): + both facts are stored. ``fail_inserts`` simulates a transiently + unavailable store. + """ + if self.fail_inserts: + raise StoreUnavailable("in-memory store is unavailable") + + inserted: list[uuid.UUID] = [] + for record in records: + existing = self._events.get(record.event_id) + if existing is not None: + if existing.digest != record.digest: + raise IntegrityConflictError( + record.event_id, existing.digest, record.digest + ) + continue + self._staged_events[record.event_id] = record + inserted.append(record.event_id) + return inserted + + def stored_events(self) -> list[ResearchEventRecord]: + """Return the committed records (test introspection).""" + return list(self._events.values()) + + def event_count(self) -> int: + """Number of committed canonical facts.""" + return len(self._events) + + # -- receipts --------------------------------------------------------- + + def record_batch_receipt(self, receipt: BatchReceipt) -> None: + """Stage a receipt; the committed receipt for a batch is immutable.""" + if self.fail_receipts: + raise StoreUnavailable("in-memory receipt store is unavailable") + self._staged_receipts[receipt.batch_id] = receipt + + def get_receipt(self, batch_id: uuid.UUID) -> Optional[BatchReceipt]: + """Fetch the committed immutable receipt for a batch, or ``None``.""" + return self._receipts.get(batch_id) + + # -- transaction boundary -------------------------------------------- + + def commit(self) -> None: + """Make every staged write durable, atomically.""" + if self.fail_commit: + raise StoreUnavailable("in-memory commit failed") + # The batch id is unique: a committed receipt for the same batch wins + # and this commit must not replace it. + for batch_id, receipt in self._staged_receipts.items(): + existing = self._receipts.get(batch_id) + if existing is not None and existing.receipt_id != receipt.receipt_id: + raise ReceiptConflictError(batch_id) + for key, record in self._staged_events.items(): + self._events.setdefault(key, record) + for batch_id, receipt in self._staged_receipts.items(): + self._receipts.setdefault(batch_id, receipt) + self._staged_events.clear() + self._staged_receipts.clear() + + def rollback(self) -> None: + """Discard every staged write, leaving committed state unchanged.""" + self._staged_events.clear() + self._staged_receipts.clear() + + # -- emitter cursors -------------------------------------------------- + + def get_emitter_cursor( + self, emitter_id: str, research_session_id: Optional[uuid.UUID] + ) -> Optional[int]: + """Return the max committed sequence for an emitter, or ``None``.""" + sequences = [ + record.emitter_sequence + for record in self._events.values() + if record.emitter_id == emitter_id + and record.research_session_id == research_session_id + ] + return max(sequences) if sequences else None + + +def row_to_record(row: ResearchEvent) -> ResearchEventRecord: + """Rehydrate a stored event row into the record model. + + The complete ``envelope_json`` document is the authority; the searchable + scalar columns are read alongside it (they are derived from the same envelope + at insert time). + """ + data = { + name: getattr(row, name) + for name in ResearchEventRecord.model_fields + if name != "envelope" + } + data["envelope"] = row.envelope_json or {} + return ResearchEventRecord.model_validate(data) + + +class SqlAlchemyIngestionStore: + """SQLAlchemy-backed ingestion store with an explicit transaction boundary.""" + + def __init__(self, session: Session) -> None: + self.session = session + + def get_stored_digests( + self, event_ids: Sequence[uuid.UUID] + ) -> dict[uuid.UUID, str]: + """Return ``{event_id: digest}`` for already-stored event ids.""" + if not event_ids: + return {} + statement = select(ResearchEvent.event_id, ResearchEvent.digest).where( + ResearchEvent.event_id.in_(list(event_ids)) + ) + return {row[0]: row[1] for row in self.session.execute(statement).all()} + + def insert_events(self, records: Sequence[ResearchEventRecord]) -> list[uuid.UUID]: + """Stage insert-only event persistence (unique ``event_id``). + + ``ON CONFLICT DO NOTHING`` makes a concurrent duplicate a no-op rather + than aborting the transaction, so the digest-identity check (performed + inside the same transaction) remains authoritative and an existing fact + is never overwritten. ``event_id`` is the only unique key on the table + (C-01 dropped the emitter-sequence constraint), so any other integrity + error is a genuine store failure and is reported retryable. + """ + if not records: + return [] + try: + values = [ + { + **record.model_dump(exclude={"envelope"}), + "envelope_json": record.envelope, + } + for record in records + ] + statement = pg_insert(ResearchEvent).values(values) + statement = statement.on_conflict_do_nothing( + index_elements=["event_id"] + ) + self.session.execute(statement) + except IntegrityError as error: + self.session.rollback() + # A persistent violation (an FK after manual data surgery, ...) would + # otherwise retry forever showing only STORE_UNAVAILABLE in the ack log. + logger.warning("research_event insert hit an integrity constraint: %s", error) + raise StoreUnavailable("event store is unavailable") from error + except SQLAlchemyError as error: + raise StoreUnavailable("event store is unavailable") from error + return [record.event_id for record in records] + + def record_batch_receipt(self, receipt: BatchReceipt) -> None: + """Stage a receipt if the batch has none (immutable, retry-safe). + + ``ON CONFLICT DO NOTHING`` on the unique ``batch_id`` means a concurrent + commit for the same batch never replaces the winner's receipt; the + service re-reads the stored receipt after commit and returns that one. + """ + try: + statement = ( + pg_insert(TelemetryBatchReceipt) + .values( + receipt_id=receipt.receipt_id, + batch_id=str(receipt.batch_id), + enrollment_id=receipt.enrollment_id, + research_session_id=receipt.research_session_id, + accepted_at=receipt.accepted_at, + receipt_json=receipt.ack.model_dump(mode="json"), + ) + .on_conflict_do_nothing(index_elements=["batch_id"]) + ) + self.session.execute(statement) + except SQLAlchemyError as error: + raise StoreUnavailable("receipt store is unavailable") from error + + def commit(self) -> None: + """Commit the whole batch (events + receipt) atomically.""" + try: + self.session.commit() + except IntegrityError as error: + self.session.rollback() + # A concurrent same-batch commit is handled with ON CONFLICT DO + # NOTHING, so an integrity error here is a genuine store failure + # (for example a globally reused event id): never a false accept. + raise StoreUnavailable( + "event insert hit a database integrity constraint" + ) from error + except SQLAlchemyError as error: + self.session.rollback() + raise StoreUnavailable("event store is unavailable") from error + + def rollback(self) -> None: + """Discard the current transaction.""" + self.session.rollback() + + def get_receipt(self, batch_id: uuid.UUID) -> Optional[BatchReceipt]: + """Fetch the immutable receipt for a batch, or ``None``.""" + statement = select(TelemetryBatchReceipt).where( + TelemetryBatchReceipt.batch_id == str(batch_id) + ) + row = self.session.execute(statement).scalars().first() + if row is None: + return None + from .models import TelemetryBatchAckV1 + + return BatchReceipt( + receipt_id=row.receipt_id, + batch_id=uuid.UUID(str(row.batch_id)), + enrollment_id=row.enrollment_id, + research_session_id=row.research_session_id, + accepted_at=row.accepted_at, + ack=TelemetryBatchAckV1.model_validate(row.receipt_json), + ) + + def get_emitter_cursor( + self, emitter_id: str, research_session_id: Optional[uuid.UUID] + ) -> Optional[int]: + """Return the max stored sequence for an emitter, or ``None``. + + The cursor is a pure projection of the event store, so no cursor row is + persisted: ``max(emitter_sequence)`` for the ``(emitter, session)``. + """ + statement = select(func.max(ResearchEvent.emitter_sequence)).where( + ResearchEvent.emitter_id == emitter_id, + ResearchEvent.research_session_id == research_session_id, + ) + value = self.session.execute(statement).scalar() + return int(value) if value is not None else None + + +def get_receipt_by_id( + session: Session, receipt_id: uuid.UUID +) -> Optional[BatchReceipt]: + """Fetch an immutable receipt by its receipt id, or ``None``.""" + statement = select(TelemetryBatchReceipt).where( + TelemetryBatchReceipt.receipt_id == receipt_id + ) + row = session.execute(statement).scalars().first() + if row is None: + return None + from .models import TelemetryBatchAckV1 + + return BatchReceipt( + receipt_id=row.receipt_id, + batch_id=uuid.UUID(str(row.batch_id)), + enrollment_id=row.enrollment_id, + research_session_id=row.research_session_id, + accepted_at=row.accepted_at, + ack=TelemetryBatchAckV1.model_validate(row.receipt_json), + ) diff --git a/src/research/telemetry/ingestion/validation.py b/src/research/telemetry/ingestion/validation.py new file mode 100644 index 0000000..efb7906 --- /dev/null +++ b/src/research/telemetry/ingestion/validation.py @@ -0,0 +1,166 @@ +"""Per-event validation for ingestion batches. + +Validation never repairs or reclassifies an event. It returns typed +:class:`~research.telemetry.ingestion.models.IngestionIssue` findings: schema/version/ +provenance/context problems and a defense-in-depth sensitive-field scan. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +from research.telemetry.validation import ( + TelemetryValidationCode, + validate_canonical_event, +) + +from .enums import IngestionReasonCode +from .models import IngestionContext, IngestionIssue + +if TYPE_CHECKING: + from research.telemetry.models import CanonicalEventV1 + +__all__ = ["validate_batch_event"] + +_SCHEMA_VERSION = "1" + +_CODE_MAP: dict[TelemetryValidationCode, IngestionReasonCode] = { + TelemetryValidationCode.SCHEMA_INVALID: IngestionReasonCode.INVALID_SCHEMA, + TelemetryValidationCode.SCHEMA_VERSION_UNSUPPORTED: ( + IngestionReasonCode.UNKNOWN_SCHEMA_VERSION + ), + TelemetryValidationCode.SCHEMA_VERSION_MISSING: ( + IngestionReasonCode.UNKNOWN_SCHEMA_VERSION + ), + TelemetryValidationCode.MISSING_EVENT_ID: IngestionReasonCode.MISSING_EVENT_ID, + TelemetryValidationCode.MISSING_PROVENANCE: IngestionReasonCode.MISSING_PROVENANCE, + TelemetryValidationCode.MISSING_EMITTER_ID: IngestionReasonCode.INVALID_SCHEMA, + TelemetryValidationCode.INVALID_EMITTER_SEQUENCE: IngestionReasonCode.INVALID_SCHEMA, + TelemetryValidationCode.MISSING_OCCURRED_AT: IngestionReasonCode.INVALID_SCHEMA, + TelemetryValidationCode.SECRET_PRESENT: IngestionReasonCode.SENSITIVE_PAYLOAD, + TelemetryValidationCode.PRIVACY_BLOCKED: IngestionReasonCode.PRIVACY_BLOCKED, +} + + +def _issue( + code: IngestionReasonCode, field: str, message: str +) -> IngestionIssue: + return IngestionIssue(code=code, field=field, message=message, permanent=True) + + +def validate_batch_event( + event: CanonicalEventV1, + context: Optional[IngestionContext], + *, + telemetry_schema_version: str = _SCHEMA_VERSION, +) -> list[IngestionIssue]: + """Return every permanent reason ``event`` cannot be stored.""" + issues: list[IngestionIssue] = [] + + if event.schema_version != telemetry_schema_version: + issues.append( + _issue( + IngestionReasonCode.UNKNOWN_SCHEMA_VERSION, + "schema_version", + ( + f"event schema_version {event.schema_version!r} does not match " + f"the batch telemetry_schema_version {telemetry_schema_version!r}" + ), + ) + ) + elif event.schema_version != _SCHEMA_VERSION: + issues.append( + _issue( + IngestionReasonCode.UNKNOWN_SCHEMA_VERSION, + "schema_version", + f"unsupported telemetry schema version {event.schema_version!r}", + ) + ) + + for finding in validate_canonical_event(event): + if finding.code is TelemetryValidationCode.UNKNOWN_EVENT_TYPE: + # A preserved marker (``unknown_event_type`` set, event_type + # ``unknown_source_event``) stays accepted with NEEDS_REVIEW; a raw + # unrecognized type with no marker is permanently rejected (TSCH-01). + if event.unknown_event_type is None: + issues.append( + _issue( + IngestionReasonCode.UNKNOWN_EVENT_TYPE, + finding.field, + finding.message, + ) + ) + continue + if finding.code is TelemetryValidationCode.UNKNOWN_SOURCE: + # Same rule for the source vocabulary (TSCH-02). + if event.unknown_source is None: + issues.append( + _issue( + IngestionReasonCode.UNKNOWN_SOURCE, + finding.field, + finding.message, + ) + ) + continue + mapped = _CODE_MAP.get(finding.code) + if mapped is None: + # Other ``needs_review`` findings are preserved, not rejected. + continue + issues.append(_issue(mapped, finding.field, finding.message)) + + if not event.provenance: + issues.append( + _issue( + IngestionReasonCode.MISSING_PROVENANCE, + "provenance", + "provenance is required to persist a canonical event", + ) + ) + elif ( + event.provenance.source + and event.source + and event.provenance.source != event.source + ): + # The builder always sets the two equal; a divergent wire value is a + # consistency failure, not a second source (TSCH-02). + issues.append( + _issue( + IngestionReasonCode.INVALID_SCHEMA, + "provenance.source", + "provenance.source must match the top-level source", + ) + ) + + if context is not None: + if event.study_id is not None and event.study_id != context.study_id: + issues.append( + _issue( + IngestionReasonCode.CONTEXT_MISMATCH, + "study_id", + "event study does not match the authorized study", + ) + ) + if ( + event.enrollment_id is not None + and event.enrollment_id != context.enrollment_id + ): + issues.append( + _issue( + IngestionReasonCode.CONTEXT_MISMATCH, + "enrollment_id", + "event enrollment does not match the authorized enrollment", + ) + ) + if ( + event.research_session_id is not None + and event.research_session_id != context.research_session_id + ): + issues.append( + _issue( + IngestionReasonCode.SESSION_OUT_OF_SCOPE, + "research_session_id", + "event belongs to a different research session", + ) + ) + + return issues diff --git a/src/research/telemetry/models.py b/src/research/telemetry/models.py new file mode 100644 index 0000000..dafce87 --- /dev/null +++ b/src/research/telemetry/models.py @@ -0,0 +1,163 @@ +"""Pydantic v2 contracts for the canonical telemetry envelope (``CanonicalEventV1``). + +Design rules encoded here: + +* An event is immutable after construction (``frozen=True``). +* ``usage_tokens`` is nullable and is paired with an explicit + :class:`Coverage` ``usage_capability``: ``0`` means an observed zero and + ``None`` means "not exposed". The two are never conflated. +* Unknown enum/input values are preserved on ``unknown_*`` fields and flagged + with a ``NEEDS_REVIEW`` coverage state rather than reclassified. +* ``SECRET`` material is never persistable; the privacy engine removes it + before serialization. The model itself carries only sanitized payload. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Any, Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import CanonicalFidelity, CoverageState + +_FROZEN = ConfigDict(extra="forbid", frozen=True) + + +class Provenance(BaseModel): + """Where a canonical event came from and how faithfully it was derived.""" + + model_config = _FROZEN + + source: str + source_event_id: Optional[str] = None + normalizer_version: str + adapter_version: Optional[str] = None + fidelity: CanonicalFidelity = CanonicalFidelity.NORMALIZED + evidence_digest: Optional[str] = None + + +class PrivacySummary(BaseModel): + """What the privacy filter did to one event.""" + + model_config = _FROZEN + + policy_digest: Optional[str] = None + actions: dict[str, int] = Field(default_factory=dict) + redacted_fields: list[str] = Field(default_factory=list) + blocked: bool = False + block_reason: Optional[str] = None + field_classes: dict[str, int] = Field(default_factory=dict) + + +class Coverage(BaseModel): + """Coverage of one measured/observed field or capability.""" + + model_config = _FROZEN + + state: CoverageState = CoverageState.UNKNOWN + reason: Optional[str] = None + capability: Optional[str] = None + + +class EventMetrics(BaseModel): + """Numeric measurements attached to an event. + + ``usage_tokens`` is ``Optional`` and paired with ``usage_capability``: + an observed zero is ``0`` with ``AVAILABLE`` coverage, while an unexposed + count is ``None`` with ``UNAVAILABLE`` coverage. + """ + + model_config = _FROZEN + + usage_tokens: Optional[int] = None + # A bare event exposes no usage measurement, so the coverage default is + # UNAVAILABLE (never the ambiguous UNKNOWN): an unexposed count is ``None``. + usage_capability: Coverage = Field( + default_factory=lambda: Coverage(state=CoverageState.UNAVAILABLE) + ) + latency_ms: Optional[int] = None + counts: dict[str, int] = Field(default_factory=dict) + + +class Correlations(BaseModel): + """Opaque correlation handles that tie related events together.""" + + model_config = _FROZEN + + turn_id: Optional[str] = None + tool_call_id: Optional[str] = None + permission_id: Optional[str] = None + edit_id: Optional[str] = None + correlation_id: Optional[str] = None + # The runtime's own message identifier, when the producer reports one. + # Distinct from span_id (transport span) and correlation_id (request + # scope): it names the logical message within the runtime's protocol. + message_id: Optional[str] = None + # Distributed-tracing handles. ``trace_id`` scopes one end-to-end trace + # (an agent run); ``span_id`` identifies this event's own span and + # ``parent_span_id`` its parent, so exact parent-child graphs are + # reconstructible without trusting timestamps. ``model_call_id`` is the + # stable handle linking one model invocation (start → completion) with the + # tool calls it caused. All are opaque strings, never content. + trace_id: Optional[str] = None + span_id: Optional[str] = None + parent_span_id: Optional[str] = None + model_call_id: Optional[str] = None + + +class CanonicalEventV1(BaseModel): + """One immutable, privacy-filtered canonical telemetry event.""" + + model_config = _FROZEN + + event_id: UUID + # Required: the envelope version is never silently defaulted on ingestion. + # A missing version is a typed schema error, not an implicit "1". + schema_version: str + # Canonical dotted event type, or "unknown_source_event" when the source + # construct is not documented. The original value is kept in + # ``unknown_event_type``. + event_type: str + source: str + + study_id: Optional[UUID] = None + enrollment_id: Optional[UUID] = None + research_session_id: Optional[UUID] = None + agent_run_id: Optional[str] = None + + occurred_at: datetime + # Per-emitter monotonic clock reading, if the emitter provided one. Global + # ordering is a query projection, never a stored fact. + monotonic_ns: Optional[int] = None + + emitter_id: str + emitter_sequence: int + + correlations: Correlations = Field(default_factory=Correlations) + lifecycle_state: Optional[str] = None + + # Sanitized, privacy-filtered mapping. Unknown fields are prefixed + # ``unknown_`` and never reclassified into a known concept. + payload: dict[str, Any] = Field(default_factory=dict) + + metrics: EventMetrics = Field(default_factory=EventMetrics) + privacy: PrivacySummary = Field(default_factory=PrivacySummary) + provenance: Provenance + coverage: Coverage = Field(default_factory=Coverage) + + # Preserved, un-reclassified inputs. + unknown_event_type: Optional[str] = None + unknown_source: Optional[str] = None + unknown_lifecycle_state: Optional[str] = None + + @property + def needs_review(self) -> bool: + """Whether any part of this event was not recognized and needs review.""" + return ( + self.coverage.state == CoverageState.NEEDS_REVIEW + or self.unknown_event_type is not None + or self.unknown_source is not None + or self.unknown_lifecycle_state is not None + ) diff --git a/src/research/telemetry/normalization/__init__.py b/src/research/telemetry/normalization/__init__.py new file mode 100644 index 0000000..940dc1b --- /dev/null +++ b/src/research/telemetry/normalization/__init__.py @@ -0,0 +1,101 @@ +"""Source normalizers that map observations into canonical candidates. + +* :mod:`research.telemetry.normalization.generic_acp` - the built-in ACP + normalizer (plus a sanitized ``unknown_source_event`` fallback and the optional + :class:`~research.telemetry.normalization.generic_acp.AgentAdapter` enrichment + interface). +* :mod:`research.telemetry.normalization.adapters` - the static, server-side + allowlist that resolves an optional adapter by its ``AdapterRef.adapter_id`` + and the :func:`normalize_acp_observation` seam (generic result unchanged when + no adapter resolves). + +Normalizers are pure: :func:`materialize_candidate` attaches the event id and +per-emitter sequence through the :class:`~research.telemetry.builder.EventBuilder`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +from .adapters import ( + AdapterSpec, + clear_adapters_for_tests, + normalize_acp_observation, + register_adapter, + registered_adapter_ids, + registered_adapters, + resolve_adapter, + unregister_adapter, +) +from .generic_acp import ( + GENERIC_ACP_NORMALIZER_VERSION, + AgentAdapter, + GenericAcpNormalizer, + enrich_with_adapter, +) +from .models import CanonicalCandidateV1, NormalizationResultV1 + +if TYPE_CHECKING: + from datetime import datetime + from uuid import UUID + + from ..builder import EventBuilder + from ..models import CanonicalEventV1 + +__all__ = [ + "GENERIC_ACP_NORMALIZER_VERSION", + "AdapterSpec", + "AgentAdapter", + "CanonicalCandidateV1", + "GenericAcpNormalizer", + "NormalizationResultV1", + "clear_adapters_for_tests", + "enrich_with_adapter", + "materialize_candidate", + "normalize_acp_observation", + "register_adapter", + "registered_adapter_ids", + "registered_adapters", + "resolve_adapter", + "unregister_adapter", +] + + +def materialize_candidate( + builder: EventBuilder, + candidate: CanonicalCandidateV1, + result: NormalizationResultV1, + *, + emitter_id: str, + occurred_at: datetime, + study_id: Optional[UUID] = None, + enrollment_id: Optional[UUID] = None, + research_session_id: Optional[UUID] = None, + agent_run_id: Optional[str] = None, + monotonic_ns: Optional[int] = None, + event_id: Optional[UUID] = None, + emitter_sequence: Optional[int] = None, +) -> CanonicalEventV1: + """Build an immutable canonical event from a candidate and its result.""" + return builder.build( + emitter_id=emitter_id, + event_type=candidate.event_type, + source=result.source, + occurred_at=occurred_at, + normalizer_version=result.normalizer_version, + payload=candidate.payload, + fidelity=candidate.fidelity, + adapter_version=candidate.adapter_version or result.adapter_version, + source_event_id=result.source_event_id, + metrics=candidate.metrics, + coverage=candidate.coverage, + lifecycle_state=candidate.lifecycle_state, + correlations=candidate.correlations, + monotonic_ns=monotonic_ns, + study_id=study_id, + enrollment_id=enrollment_id, + research_session_id=research_session_id, + agent_run_id=agent_run_id, + event_id=event_id, + emitter_sequence=emitter_sequence, + ) diff --git a/src/research/telemetry/normalization/adapters.py b/src/research/telemetry/normalization/adapters.py new file mode 100644 index 0000000..33b4ae4 --- /dev/null +++ b/src/research/telemetry/normalization/adapters.py @@ -0,0 +1,269 @@ +"""Static, server-side allowlist for optional telemetry enrichment adapters. + +Adapter seam recorded from the Task 09 decisions (agent runtime extensibility +and Codex): + +1. Codex speaks ACP through the generic path. Its observations are normalized + by :class:`~research.telemetry.normalization.generic_acp.GenericAcpNormalizer`; + no second, vendor-specific telemetry schema is introduced. +2. Distribution for the Codex integration is ``BYOA_EXTERNAL`` only. The + ``PACKAGED`` branch remains for the managed ``code4me2-agent`` release. +3. Telemetry stays generic-only, with this *allowlisted* adapter seam as the + single optional enrichment point. There is no Codex-specific enrichment + enabled now; an adapter can only add fields, never rewrite or delete the + generic canonical event (see + :func:`~research.telemetry.normalization.generic_acp.enrich_with_adapter`). +4. Adapters are resolved from a **static server-side allowlist keyed by** + ``AdapterRef.adapter_id``. An unknown or absent id falls back to the generic + result. A caller-provided module path, dotted class name or factory is never + imported, evaluated or otherwise executed; only an id already registered in + this process can ever produce an adapter. + +The registry is empty by default, so ``resolve_adapter`` returns ``None`` and +``normalize_acp_observation`` returns the untouched generic result. A test or a +trusted deployment registration helper populates it explicitly. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any, Callable, Optional + +from .generic_acp import AgentAdapter, GenericAcpNormalizer, enrich_with_adapter +from .models import NormalizationResultV1 +from research.telemetry.privacy import PrivacyPolicy, filter_payload + +__all__ = [ + "AdapterSpec", + "clear_adapters_for_tests", + "normalize_acp_observation", + "register_adapter", + "registered_adapter_ids", + "registered_adapters", + "resolve_adapter", + "unregister_adapter", +] + +#: An adapter factory is invoked with no arguments. It is registered +#: server-side; no client input can ever reach it. +AdapterFactory = Callable[[], AgentAdapter] + +_VERSION_TOKEN = re.compile(r"\d+") +_RANGE_CLAUSE = re.compile(r"^\s*(>=|<=|==|>|<|=)?\s*([0-9][0-9A-Za-z.\-+]*)\s*$") + + +@dataclass(frozen=True) +class AdapterSpec: + """Allowlist metadata for one registered adapter. + + ``supported_release_ranges`` is an immutable tuple of range expressions + (for example ``">=1.2.0,<1.3.0"``) the adapter claims to drive. A release + version outside every declared range resolves to no adapter, i.e. the + generic fallback. + """ + + adapter_id: str + adapter_version: str + supported_release_ranges: tuple[str, ...] = () + + def __post_init__(self) -> None: + if not self.adapter_id.strip(): + raise ValueError("adapter_id must not be blank") + if not self.adapter_version.strip(): + raise ValueError("adapter_version must not be blank") + for expression in self.supported_release_ranges: + if not expression.strip(): + raise ValueError("supported_release_ranges entries must not be blank") + + +@dataclass(frozen=True) +class _Registration: + spec: AdapterSpec + factory: AdapterFactory + + +#: The process-wide allowlist. It is intentionally private: callers may only +#: add/remove entries through the helpers below, and never through a name that +#: came from request input. +_REGISTRY: dict[str, _Registration] = {} + + +def register_adapter( + spec: AdapterSpec, + factory: AdapterFactory, + *, + replace: bool = False, +) -> None: + """Add one allowlisted adapter. + + Raises :class:`ValueError` when the id is already registered and + ``replace`` is false, so an accidental duplicate can never silently + shadow an approved adapter. + """ + if not callable(factory): + raise TypeError("factory must be callable") + if spec.adapter_id in _REGISTRY and not replace: + raise ValueError(f"adapter {spec.adapter_id!r} is already registered") + _REGISTRY[spec.adapter_id] = _Registration(spec=spec, factory=factory) + + +def unregister_adapter(adapter_id: str) -> bool: + """Remove one allowlisted adapter; returns whether it existed.""" + return _REGISTRY.pop((adapter_id or "").strip(), None) is not None + + +def registered_adapter_ids() -> tuple[str, ...]: + """The allowlisted adapter ids, in a stable order.""" + return tuple(sorted(_REGISTRY)) + + +def registered_adapters() -> tuple[AdapterSpec, ...]: + """The allowlisted adapter specs, in a stable order.""" + return tuple(_REGISTRY[adapter_id].spec for adapter_id in registered_adapter_ids()) + + +def clear_adapters_for_tests() -> None: + """Drop every registration. + + Test-only isolation helper: the production allowlist is static, so no + runtime path should call this. + """ + _REGISTRY.clear() + + +def _adapter_id_of(adapter_ref_or_id: Any) -> Optional[str]: + """Extract a plain adapter id from a ref/string without trusting anything else.""" + if adapter_ref_or_id is None: + return None + if isinstance(adapter_ref_or_id, str): + candidate = adapter_ref_or_id.strip() + return candidate or None + attribute = getattr(adapter_ref_or_id, "adapter_id", None) + if isinstance(attribute, str): + candidate = attribute.strip() + return candidate or None + return None + + +def _version_tuple(value: str) -> tuple[int, ...]: + return tuple(int(part) for part in _VERSION_TOKEN.findall(str(value))) + + +def _clause_includes(clause: str, version: str) -> bool: + match = _RANGE_CLAUSE.match(clause) + if match is None: + # An unparseable clause must not silently widen the allowlist. + return False + operator = match.group(1) or "==" + bound = match.group(2) + current = _version_tuple(version) + if not current: + return False + target = _version_tuple(bound) + if operator == ">=": + return current >= target + if operator == ">": + return current > target + if operator == "<=": + return current <= target + if operator == "<": + return current < target + return current == target + + +def _range_includes(expression: str, version: str) -> bool: + """Whether a comma-separated range expression includes ``version``.""" + clauses = [clause.strip() for clause in str(expression).split(",") if clause.strip()] + if not clauses: + return False + return all(_clause_includes(clause, version) for clause in clauses) + + +def resolve_adapter( + adapter_ref_or_id: Any, + *, + release_version: Optional[str] = None, +) -> Optional[AgentAdapter]: + """Resolve an allowlisted adapter, or ``None`` for the generic fallback. + + ``None`` is returned when no ref is supplied, when the id is unknown, or + when the release version cannot be proven compatible. Only an id that is + already present in the static allowlist can produce an adapter; the + ``adapter_ref_or_id`` value is used for its id alone and is never imported + or evaluated. + + Version activation is fail-closed and uses the **registered** spec's ranges + only: a caller-supplied ref can never widen (or narrow) the allowlisted + window, and a registration that declares no usable range cannot be + activated for a known version. + """ + adapter_id = _adapter_id_of(adapter_ref_or_id) + if adapter_id is None: + return None + registration = _REGISTRY.get(adapter_id) + if registration is None: + return None + if release_version is not None: + ranges = [ + str(item) + for item in registration.spec.supported_release_ranges + if str(item).strip() + ] + if not ranges: + # No declared compatibility window: refuse rather than activate an + # unverifiable adapter version (the caller falls back to generic). + return None + if not any(_range_includes(expression, release_version) for expression in ranges): + return None + return registration.factory() + + +def normalize_acp_observation( + observation: Any, + *, + adapter_ref: Any = None, + release_version: Optional[str] = None, + normalizer: Optional[GenericAcpNormalizer] = None, + source_event_id: Optional[str] = None, + direction: Optional[str] = None, + privacy_policy: Optional[PrivacyPolicy] = None, +) -> NormalizationResultV1: + """Normalize one raw ACP observation, then optionally enrich it. + + The generic ACP result is authoritative: it is returned **unchanged** when + no adapter resolves. When an adapter does resolve, every candidate is run + through + :func:`~research.telemetry.normalization.generic_acp.enrich_with_adapter`, + so the generic payload still wins on conflict and the adapter version is + recorded on the enriched candidate and the result. + + Adapter-injected fields are not trusted: after enrichment every candidate + payload is re-run through the privacy filter (deny-by-default when no + ``privacy_policy`` is supplied), so a vendor adapter cannot widen what + leaves the normalizer. The generic path is untouched. + + A normalizer instance holds bounded per-stream correlation state, so callers + observing a stream must pass the same ``normalizer`` across calls. + """ + engine = normalizer if normalizer is not None else GenericAcpNormalizer() + result = engine.normalize( + observation, + source_event_id=source_event_id, + direction=direction, + ) + adapter = resolve_adapter(adapter_ref, release_version=release_version) + if adapter is None: + return result + enriched = [enrich_with_adapter(candidate, adapter) for candidate in result.candidates] + policy = privacy_policy if privacy_policy is not None else PrivacyPolicy.default() + filtered = [] + for candidate in enriched: + payload, _summary = filter_payload(candidate.payload, policy) + filtered.append(candidate.model_copy(update={"payload": payload})) + return result.model_copy( + update={ + "candidates": filtered, + "adapter_version": getattr(adapter, "adapter_version", None), + } + ) diff --git a/src/research/telemetry/normalization/generic_acp.py b/src/research/telemetry/normalization/generic_acp.py new file mode 100644 index 0000000..3e4e475 --- /dev/null +++ b/src/research/telemetry/normalization/generic_acp.py @@ -0,0 +1,1037 @@ +"""Built-in ACP source normalizer: observations into canonical concepts. + +This module hosts the built-in, pure ACP normalizer: + +* :class:`GenericAcpNormalizer` - ACP JSON-RPC messages. It maps **only + documented, stable constructs**. It is not a copy of the ACP method + vocabulary: a method/notification becomes one of the canonical + :class:`~research.telemetry.enums.CanonicalEventType` concepts, and an unknown + method becomes ``unknown_source_event`` carrying sanitized metadata (method + name and parameter *names*, never values) with a ``NEEDS_REVIEW`` coverage + state. + +Vendor-specific semantic enrichment is intentionally out of scope for the ACP +normalizer; an :class:`AgentAdapter` may enrich a candidate later without +deleting or replacing the generic observation. IDE canonicalization belongs to +the IntelliJ plugin, not the server. + +Correlation note: ACP is a request/response protocol whose updates arrive as +notifications. Two canonical facts therefore need *stream* context, not a +single frame: + +* ``permission.decided`` is derived from the **host response** to a previously + observed ``session/request_permission`` (the response only carries the + selected option id, so the request's options are needed to resolve the option + *kind*); +* ``agent.message.completed`` is derived from the response to a previously + observed ``session/prompt`` request; +* ``interaction.started`` for ``session/new`` / ``session/fork`` is derived from + the agent's successful response, the first frame that names the new chat; +* while a ``session/load`` request is pending, the agent replays the chat's + history as ``session/update`` notifications. Those were recorded when they + happened, so they are dropped rather than recorded twice. + +Those correlations (and the "``tool.created`` once per tool call" rule) are held +as bounded, per-session state on the normalizer instance. Every other mapping is +a pure function of one message. The state never changes an observation's +identity, order, or fidelity. + +Candidates derived from a mapped ACP method carry it as ``acp_method`` +(:data:`~research.telemetry.chat_lifecycle.ACP_METHOD_KEY`): readers tell chat +starts, chat ends and user interrupts apart by it. +""" + +from __future__ import annotations + +from itertools import islice +from typing import Any, Mapping, Optional, Protocol + +from ..chat_lifecycle import ACP_METHOD_KEY, CHAT_END_METHODS +from ..enums import CanonicalEventType, CanonicalFidelity, CoverageState, EventSource +from ..models import Correlations, Coverage, EventMetrics +from .models import CanonicalCandidateV1, NormalizationResultV1 + +GENERIC_ACP_NORMALIZER_VERSION = "generic-acp-v2" + +#: Canonical ACP permission option kinds that mean "allow". +_ALLOW_OPTION_KINDS = frozenset({"allow_once", "allow_always"}) +#: Canonical ACP permission option kinds that mean "reject". +_REJECT_OPTION_KINDS = frozenset({"reject_once", "reject_always"}) + +#: Requests mapped from the frame alone. A chat end (``CHAT_END_METHODS``) is +#: ``interaction.completed`` with lifecycle ``completed``; ``session/cancel``, a +#: user interrupt, keeps no lifecycle state. +_METHOD_RULES: dict[str, tuple[str, CanonicalEventType]] = { + "initialize": ("acp.initialize", CanonicalEventType.INTERACTION_STARTED), + "session/load": ("acp.session.load", CanonicalEventType.INTERACTION_STARTED), + "session/resume": ("acp.session.resume", CanonicalEventType.INTERACTION_STARTED), + "session/prompt": ("acp.session.prompt", CanonicalEventType.AGENT_MESSAGE_STARTED), + "session/cancel": ("acp.session.cancel", CanonicalEventType.INTERACTION_COMPLETED), + "session/close": ("acp.session.close", CanonicalEventType.INTERACTION_COMPLETED), + "session/delete": ("acp.session.delete", CanonicalEventType.INTERACTION_COMPLETED), + "fs/read_text_file": ("acp.fs.read_text_file", CanonicalEventType.IDE_FILE_OPENED), + "fs/write_text_file": ("acp.fs.write_text_file", CanonicalEventType.IDE_FILE_SAVED), + "terminal/create": ("acp.terminal.create", CanonicalEventType.TOOL_STARTED), +} + +#: Chat starts whose request names no chat yet. The request maps to nothing; the +#: agent's successful response, which carries the new chat's id, maps to +#: ``interaction.started``. A failed one maps to its error response alone. +_RESPONSE_CHAT_STARTS: dict[str, str] = { + "session/new": "acp.session.new", + "session/fork": "acp.session.fork", +} + +#: Bounds on the prompt one ``session/prompt`` content candidate carries. +_PROMPT_MAX_BLOCKS = 32 +_PROMPT_MAX_CHARS = 64 * 1024 + + +def _candidate( + event_type: CanonicalEventType, + rule_id: str, + *, + payload: Optional[dict[str, Any]] = None, + lifecycle_state: Optional[str] = None, + coverage: Optional[Coverage] = None, + correlations: Optional[Correlations] = None, + metrics: Optional[EventMetrics] = None, +) -> CanonicalCandidateV1: + return CanonicalCandidateV1( + event_type=event_type, + fidelity=CanonicalFidelity.NORMALIZED, + mapping_rule_id=rule_id, + payload=payload or {}, + correlations=correlations or Correlations(), + lifecycle_state=lifecycle_state, + coverage=coverage or Coverage(state=CoverageState.AVAILABLE), + metrics=metrics or EventMetrics(), + ) + + +def _with_turn_id( + candidates: list[CanonicalCandidateV1], turn_id: str +) -> list[CanonicalCandidateV1]: + """Attach [turn_id] to candidates that carry no turn of their own. + + Additive only: a candidate whose mapping already set a turn is never + overwritten. Events are frozen, so the stamp is a copy. + """ + stamped: list[CanonicalCandidateV1] = [] + for candidate in candidates: + correlations = candidate.correlations + if correlations.turn_id is not None: + stamped.append(candidate) + continue + stamped.append( + candidate.model_copy( + update={ + "correlations": correlations.model_copy( + update={"turn_id": turn_id} + ) + } + ) + ) + return stamped + + +class AgentAdapter(Protocol): + """Optional vendor adapter that enriches (never replaces) generic events.""" + + adapter_version: str + mapping_rule_version: str + supported_release_ranges: list[str] + + def enrich( + self, candidate: CanonicalCandidateV1 + ) -> Optional[CanonicalCandidateV1]: # pragma: no cover - structural protocol + ... + + +def enrich_with_adapter( + candidate: CanonicalCandidateV1, adapter: AgentAdapter +) -> CanonicalCandidateV1: + """Apply an adapter without deleting or replacing generic fields. + + The generic candidate's payload wins on conflict, so an adapter can only add + fields; it can never erase or rewrite the source observation. + """ + enriched = adapter.enrich(candidate) + if enriched is None: + return candidate + merged_payload = {**enriched.payload, **candidate.payload} + return candidate.model_copy( + update={"payload": merged_payload, "adapter_version": adapter.adapter_version} + ) + + +def _unknown_candidate( + method: Optional[str], params: Any, *, source_event_id: Optional[str] = None +) -> CanonicalCandidateV1: + payload: dict[str, Any] = {} + if method is not None: + payload["unknown_method"] = method + if isinstance(params, Mapping): + # Parameter *names* only; values may be sensitive. + payload["param_names"] = sorted(str(key) for key in params) + if source_event_id is not None: + payload["source_event_id"] = source_event_id + return _candidate( + CanonicalEventType.UNKNOWN_SOURCE_EVENT, + "acp.unknown", + payload=payload, + coverage=Coverage( + state=CoverageState.NEEDS_REVIEW, + reason="unrecognized ACP construct preserved as metadata only", + ), + ) + + +def _tool_payload(location: Mapping[str, Any]) -> dict[str, Any]: + payload: dict[str, Any] = {} + for source_key, target_key in ( + ("toolCallId", "tool_call_id"), + ("title", "tool_name"), + ("kind", "tool_kind"), + ("status", "status"), + ): + value = location.get(source_key) + if value is not None: + payload[target_key] = value + raw_input = location.get("rawInput") + if raw_input is not None: + payload["arguments"] = raw_input + content = location.get("content") + if content is not None: + payload["content"] = content + return payload + + +def _tool_call_id(update: Mapping[str, Any]) -> Optional[str]: + value = update.get("toolCallId") + return value if isinstance(value, str) and value else None + + +def _tool_correlations(update: Mapping[str, Any]) -> Correlations: + return Correlations(tool_call_id=_tool_call_id(update)) + + +def _status(update: Mapping[str, Any]) -> str: + return str(update.get("status") or "").lower() + + +def _terminal_tool_candidate( + update: Mapping[str, Any], +) -> Optional[CanonicalCandidateV1]: + """Map a terminal ``tool_call``/``tool_call_update`` status; else ``None``.""" + status = _status(update) + if status in {"completed", "success", "succeeded"}: + return _candidate( + CanonicalEventType.TOOL_COMPLETED, + "acp.update.tool_call_update.completed", + payload=_tool_payload(update), + correlations=_tool_correlations(update), + lifecycle_state="completed", + ) + if status in {"failed", "error"}: + return _candidate( + CanonicalEventType.TOOL_FAILED, + "acp.update.tool_call_update.failed", + payload=_tool_payload(update), + correlations=_tool_correlations(update), + lifecycle_state="failed", + ) + return None + + +def _first_int(source: Mapping[str, Any], keys: tuple[str, ...]) -> Optional[int]: + for key in keys: + value = source.get(key) + if isinstance(value, bool): + continue + if isinstance(value, int): + return value + return None + + +def _usage_metrics(source: Any) -> EventMetrics: + """Build usage metrics from an agent-reported usage mapping. + + A reported total (or a derivable input+output total) is an observed value + with ``AVAILABLE`` coverage - including an observed ``0``. When the agent + reports no token counts at all, the value stays ``None`` with ``UNAVAILABLE`` + coverage; a missing measurement is never coerced to zero. + """ + tokens: Optional[int] = None + if isinstance(source, Mapping): + tokens = _first_int(source, ("totalTokens", "total_tokens")) + if tokens is None: + input_tokens = _first_int(source, ("inputTokens", "input_tokens")) + output_tokens = _first_int(source, ("outputTokens", "output_tokens")) + if input_tokens is not None or output_tokens is not None: + tokens = (input_tokens or 0) + (output_tokens or 0) + if tokens is None: + tokens = _first_int(source, ("used",)) + if tokens is None: + return EventMetrics( + usage_tokens=None, + usage_capability=Coverage( + state=CoverageState.UNAVAILABLE, + reason="agent did not report token usage", + capability="usage", + ), + ) + return EventMetrics( + usage_tokens=tokens, + usage_capability=Coverage(state=CoverageState.AVAILABLE, capability="usage"), + ) + + +def _decision_for(option_id: str, kind: Optional[str]) -> str: + """Resolve an allow/reject/cancelled decision from a selected option.""" + if kind in _ALLOW_OPTION_KINDS: + return "allow" + if kind in _REJECT_OPTION_KINDS: + return "reject" + text = option_id.lower() + if "reject" in text or "deny" in text: + return "reject" + if "allow" in text or "approve" in text or "accept" in text: + return "allow" + return "selected" + + +def _normalized_options( + params: Any, +) -> tuple[list[dict[str, str]], dict[str, Optional[str]]]: + """Return ``(payload_options, {option_id: kind})`` from request params.""" + payload_options: list[dict[str, str]] = [] + option_kinds: dict[str, Optional[str]] = {} + if not isinstance(params, Mapping): + return payload_options, option_kinds + raw_options = params.get("options") + if not isinstance(raw_options, list): + return payload_options, option_kinds + for option in raw_options: + if not isinstance(option, Mapping): + continue + option_id = option.get("optionId") + if not isinstance(option_id, str): + option_id = option.get("option_id") + kind = option.get("kind") + entry: dict[str, str] = {} + if isinstance(option_id, str) and option_id: + entry["option_id"] = option_id + option_kinds[option_id] = kind if isinstance(kind, str) else None + if isinstance(kind, str): + entry["kind"] = kind + if entry: + payload_options.append(entry) + return payload_options, option_kinds + + +def _tool_call_request_tool_id(params: Any) -> Optional[str]: + if not isinstance(params, Mapping): + return None + tool_call = params.get("toolCall") + if not isinstance(tool_call, Mapping): + return None + value = tool_call.get("toolCallId") + return value if isinstance(value, str) and value else None + + +class _TextBudget: + """Characters of prompt text one content candidate may still carry.""" + + def __init__(self, limit: int) -> None: + self.remaining = limit + + def take(self, value: str) -> tuple[str, bool]: + """``value`` cut to the remaining budget, and whether it was cut.""" + kept = value[: self.remaining] + self.remaining -= len(kept) + return kept, len(kept) < len(value) + + +def _take_strings( + source: Mapping[str, Any], + keys: tuple[str, ...], + target: dict[str, Any], + budget: _TextBudget, +) -> bool: + """Copy the string values of ``keys`` within the budget; whether one was cut.""" + truncated = False + for key in keys: + value = source.get(key) + if isinstance(value, str): + target[key], cut = budget.take(value) + truncated = truncated or cut + return truncated + + +def _prompt_block( + block: Mapping[str, Any], + budget: _TextBudget, + typed: Optional[tuple[str, bool]], +) -> dict[str, Any]: + """One ACP content block with only the fields worth keeping. + + ``typed`` is the text block's already budgeted text. Binary data is reduced + to its ``data_length``; ``_meta`` and annotations are never copied. + """ + kind = block.get("type") + entry: dict[str, Any] = {"type": kind} if isinstance(kind, str) else {} + truncated = False + if typed is not None: + entry["text"], truncated = typed + elif kind == "resource_link": + truncated = _take_strings( + block, ("uri", "name", "title", "description", "mimeType"), entry, budget + ) + size = block.get("size") + if isinstance(size, int) and not isinstance(size, bool): + entry["size"] = size + elif kind == "resource" and isinstance(block.get("resource"), Mapping): + resource = block["resource"] + embedded: dict[str, Any] = {} + truncated = _take_strings(resource, ("uri", "mimeType", "text"), embedded, budget) + if isinstance(resource.get("blob"), str): + embedded["data_length"] = len(resource["blob"]) + entry["resource"] = embedded + elif kind in ("image", "audio"): + truncated = _take_strings(block, ("mimeType", "uri"), entry, budget) + if isinstance(block.get("data"), str): + entry["data_length"] = len(block["data"]) + if truncated: + entry["truncated"] = True + return entry + + +def _prompt_blocks(prompt: list[Any]) -> list[dict[str, Any]]: + """Bounded copies of a ``session/prompt``'s content blocks. + + At most ``_PROMPT_MAX_BLOCKS`` blocks and ``_PROMPT_MAX_CHARS`` characters + are kept. Typed text is what the participant asked, so it is served from + the budget first and a large attached file never crowds it out; embedded + resource text and resource links share what is left. A block that lost + text to the budget is marked ``truncated``. + """ + blocks = list( + islice((block for block in prompt if isinstance(block, Mapping)), _PROMPT_MAX_BLOCKS) + ) + budget = _TextBudget(_PROMPT_MAX_CHARS) + typed = { + index: budget.take(block["text"]) + for index, block in enumerate(blocks) + if block.get("type") == "text" and isinstance(block.get("text"), str) + } + return [_prompt_block(block, budget, typed.get(index)) for index, block in enumerate(blocks)] + + +def _prompt_content_candidate(params: Mapping[str, Any]) -> Optional[CanonicalCandidateV1]: + """The participant's prompt as a second ``agent.message.started``, if sent. + + ``message_kind`` ``user`` and lifecycle ``started`` make readers count it + with the streamed chunks, never as another turn start. ``prompt`` is + CONTENT: a metadata-only study keeps only ``[REDACTED]``. + """ + prompt = params.get("prompt") + if not isinstance(prompt, list): + return None + payload: dict[str, Any] = {ACP_METHOD_KEY: "session/prompt", "message_kind": "user"} + session_id = params.get("sessionId") + if isinstance(session_id, str): + payload["session_id"] = session_id + payload["prompt"] = _prompt_blocks(prompt) + return _candidate( + CanonicalEventType.AGENT_MESSAGE_STARTED, + "acp.session.prompt.content", + payload=payload, + lifecycle_state="started", + ) + + +class GenericAcpNormalizer: + """Maps ACP JSON-RPC messages to canonical candidates. + + A normalizer instance is bound to one observer stream (session): it keeps + only the correlation state required for ``permission.decided`` / + ``agent.message.completed`` / response-derived chat starts, for dropping a + ``session/load`` history replay, and for emitting ``tool.created`` exactly + once per tool call. + """ + + normalizer_version = GENERIC_ACP_NORMALIZER_VERSION + + def __init__(self) -> None: + #: jsonrpc permission request id -> + #: {"options": {optionId: kind}, "tool_call_id": ..., "session_id": ...} + self._pending_permissions: dict[str, dict[str, Any]] = {} + #: jsonrpc ``session/prompt`` request id -> ACP session key awaiting its response. + self._pending_prompts: dict[str, str] = {} + #: ACP session key -> the native prompt request id of its active turn. + self._current_turn: dict[str, str] = {} + #: in-flight request id -> the turn it belongs to (settled by its response). + self._request_turns: dict[str, str] = {} + #: tool call ids for which ``tool.created`` was already emitted. + self._created_tool_calls: set[str] = set() + #: jsonrpc ``session/new``/``session/fork`` request id -> that method, + #: until the agent's answer names the new chat. + self._pending_chat_starts: dict[str, str] = {} + #: jsonrpc ``session/load`` request id -> the ACP session whose history + #: the agent replays until it answers. + self._pending_loads: dict[str, str] = {} + + @staticmethod + def _session_key(message: Mapping[str, Any]) -> Optional[str]: + """The ACP session id carried by a frame, if any. + + It keys turn state when one stream multiplexes several ACP sessions. A + response may omit it; responses are matched by request id instead. + """ + for container in ("params", "result"): + section = message.get(container) + if isinstance(section, Mapping): + session_id = section.get("sessionId") + if isinstance(session_id, str) and session_id: + return session_id + return None + + def _current_turn_for(self, message: Mapping[str, Any]) -> Optional[str]: + """The active turn for this frame's session, or ``None`` when none. + + A frame without an explicit session association never inherits a turn: + no turn id is invented when no prompt is known to be pending. + """ + session_key = self._session_key(message) + if session_key is None: + return None + return self._current_turn.get(session_key) + + def _advance_turn_state(self, message: Any) -> Optional[str]: + """Update bounded turn state and return this frame's native turn id. + + The turn id is the JSON-RPC id of the ``session/prompt`` request. It is + returned for the prompt itself, for frames observed while it is pending, + and for its response; ``None`` means no prompt is pending. + """ + if not isinstance(message, Mapping): + return None + method = message.get("method") + message_id = message.get("id") + request_id = str(message_id) if message_id is not None else None + + if method == "session/prompt" and request_id is not None: + session_key = self._session_key(message) or "" + self._current_turn[session_key] = request_id + self._pending_prompts[request_id] = session_key + return request_id + + if method is None and request_id is not None: + # A response settles any tracked sub-request, else closes the prompt. + settled = self._request_turns.pop(request_id, None) + if settled is not None: + return settled + if request_id in self._pending_prompts: + return request_id + return None + + if method is None: + return None + + turn_id = self._current_turn_for(message) + if turn_id is not None and request_id is not None: + self._request_turns[request_id] = turn_id + return turn_id + + def _close_turn(self, prompt_id: str, session_key: str) -> None: + """Clear the prompt's active turn and any outstanding sub-requests.""" + if self._current_turn.get(session_key) == prompt_id: + del self._current_turn[session_key] + for request_id in [ + key for key, turn in self._request_turns.items() if turn == prompt_id + ]: + del self._request_turns[request_id] + + def _replaying(self, params: Any) -> bool: + """Whether this ``session/update`` is history a pending load replays.""" + if not isinstance(params, Mapping): + return False + session_id = params.get("sessionId") + return isinstance(session_id, str) and session_id in self._pending_loads.values() + + def _end_replay(self, session_id: Optional[str]) -> None: + """Forget pending loads of ``session_id``: its updates are live again.""" + for request_id in [ + key for key, loading in self._pending_loads.items() if loading == session_id + ]: + del self._pending_loads[request_id] + + def _settle_agent_answer( + self, response_id: str, direction: Optional[str] + ) -> Optional[str]: + """Settle a chat-start or load request the agent answered or failed. + + Returns the chat-start method the answer belongs to, if any. Any answer + to a ``session/load`` ends its replay window. + """ + if direction not in (None, "agent_to_host"): + return None + self._pending_loads.pop(response_id, None) + return self._pending_chat_starts.pop(response_id, None) + + def normalize( + self, + message: Any, + *, + source_event_id: Optional[str] = None, + direction: Optional[str] = None, + ) -> NormalizationResultV1: + """Normalize one ACP message and attach its native turn correlation. + + Turn state advances before mapping: the ``session/prompt`` request opens + a turn, frames observed while it is pending inherit its id, and the + prompt response closes it. State is per-stream (this instance), keyed by + ACP session, so concurrent sessions never share a turn. + """ + turn_id = self._advance_turn_state(message) + result = self._normalize_message( + message, + source_event_id=source_event_id, + direction=direction, + ) + if turn_id is None: + return result + return result.model_copy( + update={"candidates": _with_turn_id(result.candidates, turn_id)} + ) + + def _normalize_message( + self, + message: Any, + *, + source_event_id: Optional[str] = None, + direction: Optional[str] = None, + ) -> NormalizationResultV1: + """Map one ACP message (zero or more candidates) to canonical concepts. + + ``direction`` is the optional ``host_to_agent`` / ``agent_to_host`` frame + direction. When provided it disambiguates responses (a permission + decision is only taken from a host response; a message completion only + from an agent response); when omitted, correlation falls back to the + JSON-RPC id alone. + """ + if not isinstance(message, Mapping): + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=source_event_id, + candidates=[], + unmapped_reason="message is not a JSON object", + ) + + method = message.get("method") + params = message.get("params") + message_id = message.get("id") + resolved_source_id = ( + source_event_id + if source_event_id is not None + else (str(message_id) if message_id is not None else None) + ) + + if method == "session/update" and self._replaying(params): + # Replayed history was recorded when it happened; it is not flagged + # and recorded again, it is dropped. + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=[], + unmapped_reason="history replayed for session/load", + ) + + if method == "session/update" and isinstance(params, Mapping): + update = params.get("update") + if isinstance(update, Mapping): + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=self._update_candidates(update), + ) + + if method == "session/request_permission": + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=[self._permission_request(message_id, params)], + ) + + if isinstance(method, str) and method in _RESPONSE_CHAT_STARTS: + if message_id is not None: + self._pending_chat_starts[str(message_id)] = method + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=[], + unmapped_reason="chat start is mapped from the agent's response", + ) + + if isinstance(method, str) and method in _METHOD_RULES: + rule_id, event_type = _METHOD_RULES[method] + payload: dict[str, Any] = {ACP_METHOD_KEY: method} + session_id: Optional[str] = None + if isinstance(params, Mapping): + raw_session_id = params.get("sessionId") + if isinstance(raw_session_id, str): + session_id = raw_session_id + payload["session_id"] = session_id + candidates = [ + _candidate( + event_type, + rule_id, + payload=payload, + lifecycle_state="completed" if method in CHAT_END_METHODS else None, + ) + ] + if method == "session/load" and session_id and message_id is not None: + self._pending_loads[str(message_id)] = session_id + if method == "session/prompt" and isinstance(params, Mapping): + # A new prompt means the chat is live, even if a load's answer + # was never observed. + self._end_replay(session_id) + content = _prompt_content_candidate(params) + if content is not None: + candidates.append(content) + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=candidates, + ) + + if "error" in message: + if method is None and message_id is not None: + # A failed chat start or load maps to this error alone. + self._settle_agent_answer(str(message_id), direction) + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=[self._agent_error(message, direction)], + ) + + # A response carries no method; pair it with an observed request. + if method is None and message_id is not None: + response = self._response_candidate(str(message_id), message, direction) + if response is not None: + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=[response], + ) + + return NormalizationResultV1( + source=EventSource.ACP, + normalizer_version=self.normalizer_version, + source_event_id=resolved_source_id, + candidates=[_unknown_candidate(method, params, source_event_id=resolved_source_id)], + ) + + # -- session/update ---------------------------------------------------- + + def _update_candidates( + self, update: Mapping[str, Any] + ) -> list[CanonicalCandidateV1]: + kind = update.get("sessionUpdate") + + if kind == "agent_message_chunk": + payload: dict[str, Any] = {"message_kind": "assistant"} + if isinstance(update.get("messageId"), str): + payload["message_id"] = update["messageId"] + if update.get("content") is not None: + payload["content"] = update["content"] + return [ + _candidate( + CanonicalEventType.AGENT_MESSAGE_STARTED, + "acp.update.agent_message_chunk", + payload=payload, + lifecycle_state="started", + ) + ] + + if kind == "agent_thought_chunk": + payload = {"message_kind": "thought"} + if isinstance(update.get("messageId"), str): + payload["message_id"] = update["messageId"] + if update.get("content") is not None: + # Reasoning is CONTENT and prohibited by default; it is only kept + # when content capture is explicitly allowed AND consented. + payload["reasoning"] = update["content"] + return [ + _candidate( + CanonicalEventType.AGENT_MESSAGE_STARTED, + "acp.update.agent_thought_chunk", + payload=payload, + lifecycle_state="started", + ) + ] + + if kind == "tool_call": + return self._tool_call_candidates(update) + + if kind == "tool_call_update": + terminal = _terminal_tool_candidate(update) + if terminal is not None: + return [terminal] + return [ + _candidate( + CanonicalEventType.TOOL_STARTED, + "acp.update.tool_call_update", + payload=_tool_payload(update), + correlations=_tool_correlations(update), + lifecycle_state="started", + ) + ] + + if kind == "plan": + return [self._plan_candidate(update)] + + if kind == "usage_update": + return [ + _candidate( + CanonicalEventType.USAGE_UPDATED, + "acp.update.usage_update", + metrics=_usage_metrics(update), + ) + ] + + return [ + _unknown_candidate( + f"session/update:{kind}" if kind else "session/update", update + ) + ] + + def _tool_call_candidates( + self, update: Mapping[str, Any] + ) -> list[CanonicalCandidateV1]: + tool_call_id = _tool_call_id(update) + already_created = ( + tool_call_id is not None and tool_call_id in self._created_tool_calls + ) + if already_created: + candidates = [ + _candidate( + CanonicalEventType.TOOL_STARTED, + "acp.update.tool_call", + payload=_tool_payload(update), + correlations=_tool_correlations(update), + lifecycle_state="started", + ) + ] + else: + if tool_call_id is not None: + self._created_tool_calls.add(tool_call_id) + candidates = [ + _candidate( + CanonicalEventType.TOOL_CREATED, + "acp.update.tool_call.created", + payload=_tool_payload(update), + correlations=_tool_correlations(update), + lifecycle_state="pending", + ) + ] + # A single-shot ``tool_call`` can already be terminal. + terminal = _terminal_tool_candidate(update) + if terminal is not None: + candidates.append(terminal) + return candidates + + def _plan_candidate(self, update: Mapping[str, Any]) -> CanonicalCandidateV1: + payload: dict[str, Any] = {} + entries = update.get("entries") + if isinstance(entries, list): + payload["plan_size"] = len(entries) + status_counts: dict[str, int] = {} + for entry in entries: + if not isinstance(entry, Mapping): + continue + status = entry.get("status") + if isinstance(status, str): + status_counts[status] = status_counts.get(status, 0) + 1 + if status_counts: + # A list of {"status", "count"} records, not a status-keyed map: + # the fail-closed classifier treats status strings used as keys + # as unclassified content and would redact their values. + payload["plan_status_counts"] = [ + {"status": status, "count": count} + for status, count in sorted(status_counts.items()) + ] + return _candidate( + CanonicalEventType.PLAN_UPDATED, "acp.update.plan", payload=payload + ) + + # -- permissions ------------------------------------------------------- + + def _permission_request( + self, message_id: Any, params: Any + ) -> CanonicalCandidateV1: + permission_id = str(message_id) if message_id is not None else None + tool_call_id = _tool_call_request_tool_id(params) + options, option_kinds = _normalized_options(params) + session_id = params.get("sessionId") if isinstance(params, Mapping) else None + if not isinstance(session_id, str) or not session_id: + session_id = None + payload: dict[str, Any] = {} + if session_id is not None: + payload["session_id"] = session_id + if tool_call_id is not None: + payload["tool_call_id"] = tool_call_id + if options: + payload["options"] = options + payload["option_count"] = len(options) + if permission_id is not None: + self._pending_permissions[permission_id] = { + "tool_call_id": tool_call_id, + "options": option_kinds, + # The host's answer names no chat; it belongs to the request's. + "session_id": session_id, + } + return _candidate( + CanonicalEventType.PERMISSION_REQUESTED, + "acp.permission.request", + payload=payload, + correlations=Correlations( + permission_id=permission_id, tool_call_id=tool_call_id + ), + ) + + def _permission_decided( + self, permission_id: str, message: Mapping[str, Any] + ) -> CanonicalCandidateV1: + spec = self._pending_permissions.pop(permission_id, {}) + tool_call_id = spec.get("tool_call_id") + option_kinds: Mapping[str, Optional[str]] = spec.get("options") or {} + result = message.get("result") + payload: dict[str, Any] = {} + if spec.get("session_id") is not None: + payload["session_id"] = spec["session_id"] + if isinstance(result, Mapping): + outcome = result.get("outcome") + if isinstance(outcome, Mapping): + selected = outcome.get("outcome") + if isinstance(selected, str): + payload["outcome"] = selected + option_id = outcome.get("optionId") + if not isinstance(option_id, str): + option_id = outcome.get("option_id") + if selected == "selected" and isinstance(option_id, str): + kind = option_kinds.get(option_id) + payload["selected_option_id"] = option_id + if isinstance(kind, str): + payload["selected_option_kind"] = kind + payload["decision"] = _decision_for(option_id, kind) + elif selected == "cancelled": + payload["decision"] = "cancelled" + elif isinstance(outcome, str): + # Some ACP revisions surface the outcome string directly. + payload["outcome"] = outcome + if outcome == "cancelled": + payload["decision"] = "cancelled" + payload.setdefault("outcome", "unknown") + payload.setdefault("decision", "unknown") + return _candidate( + CanonicalEventType.PERMISSION_DECIDED, + "acp.permission.decided", + payload=payload, + correlations=Correlations( + permission_id=permission_id, tool_call_id=tool_call_id + ), + ) + + # -- responses / messages / errors ------------------------------------- + + def _response_candidate( + self, response_id: str, message: Mapping[str, Any], direction: Optional[str] + ) -> Optional[CanonicalCandidateV1]: + if response_id in self._pending_permissions and direction in ( + None, + "host_to_agent", + ): + return self._permission_decided(response_id, message) + if response_id in self._pending_prompts and direction in (None, "agent_to_host"): + session_key = self._pending_prompts.pop(response_id) + self._close_turn(response_id, session_key) + return self._message_completed(message, session_key) + chat_start = self._settle_agent_answer(response_id, direction) + if chat_start is not None: + return self._chat_started(chat_start, message) + return None + + @staticmethod + def _chat_started(method: str, message: Mapping[str, Any]) -> CanonicalCandidateV1: + """``interaction.started`` for a new or forked chat, named by the answer.""" + payload: dict[str, Any] = {ACP_METHOD_KEY: method} + result = message.get("result") + if isinstance(result, Mapping): + session_id = result.get("sessionId") + if isinstance(session_id, str) and session_id: + payload["session_id"] = session_id + return _candidate( + CanonicalEventType.INTERACTION_STARTED, + _RESPONSE_CHAT_STARTS[method], + payload=payload, + ) + + def _message_completed( + self, message: Mapping[str, Any], session_key: str + ) -> CanonicalCandidateV1: + result = message.get("result") + payload: dict[str, Any] = {} + if session_key: + # The response names no chat; it belongs to the prompt's. + payload["session_id"] = session_key + usage_source: Any = result + if isinstance(result, Mapping): + stop_reason = result.get("stopReason") + if not isinstance(stop_reason, str): + stop_reason = result.get("stop_reason") + if isinstance(stop_reason, str): + payload["stop_reason"] = stop_reason + if isinstance(result.get("usage"), Mapping): + usage_source = result["usage"] + return _candidate( + CanonicalEventType.AGENT_MESSAGE_COMPLETED, + "acp.message.completed", + payload=payload, + lifecycle_state="completed", + metrics=_usage_metrics(usage_source), + ) + + def _agent_error( + self, message: Mapping[str, Any], direction: Optional[str] + ) -> CanonicalCandidateV1: + error = message.get("error") + # A JSON-RPC error response observed on the wire is normally the agent + # rejecting a host request; a host->agent error is a host rejection. + error_source = "host" if direction == "host_to_agent" else "agent" + payload: dict[str, Any] = {"error_source": error_source} + if isinstance(error, Mapping): + if error.get("code") is not None: + payload["error_code"] = error["code"] + if error.get("message") is not None: + payload["error_message"] = error["message"] + return _candidate( + CanonicalEventType.AGENT_ERROR, "acp.error_response", payload=payload + ) + + +__all__ = [ + "AgentAdapter", + "GENERIC_ACP_NORMALIZER_VERSION", + "GenericAcpNormalizer", + "enrich_with_adapter", +] diff --git a/src/research/telemetry/normalization/models.py b/src/research/telemetry/normalization/models.py new file mode 100644 index 0000000..a3bc930 --- /dev/null +++ b/src/research/telemetry/normalization/models.py @@ -0,0 +1,47 @@ +"""Shared normalization contracts. + +A normalizer emits zero or more :class:`CanonicalCandidateV1` objects wrapped in +a :class:`NormalizationResultV1`. Candidates are pre-build: the +:class:`~research.telemetry.builder.EventBuilder` allocates the immutable event +id and per-emitter sequence, so normalizers stay pure and deterministic. +""" + +from __future__ import annotations + +from typing import Any, Optional + +from pydantic import BaseModel, ConfigDict, Field + +from ..enums import CanonicalEventType, CanonicalFidelity, EventSource +from ..models import Correlations, Coverage, EventMetrics + +_BASE_CONFIG = ConfigDict(extra="forbid") + + +class CanonicalCandidateV1(BaseModel): + """A pre-build canonical candidate produced by a normalizer.""" + + model_config = _BASE_CONFIG + + event_type: CanonicalEventType + fidelity: CanonicalFidelity = CanonicalFidelity.NORMALIZED + mapping_rule_id: str + payload: dict[str, Any] = Field(default_factory=dict) + correlations: Correlations = Field(default_factory=Correlations) + lifecycle_state: Optional[str] = None + coverage: Coverage = Field(default_factory=Coverage) + metrics: EventMetrics = Field(default_factory=EventMetrics) + adapter_version: Optional[str] = None + + +class NormalizationResultV1(BaseModel): + """The result of normalizing one source observation.""" + + model_config = _BASE_CONFIG + + source: EventSource + normalizer_version: str + source_event_id: Optional[str] = None + adapter_version: Optional[str] = None + candidates: list[CanonicalCandidateV1] = Field(default_factory=list) + unmapped_reason: Optional[str] = None diff --git a/src/research/telemetry/privacy/__init__.py b/src/research/telemetry/privacy/__init__.py new file mode 100644 index 0000000..e7ad31d --- /dev/null +++ b/src/research/telemetry/privacy/__init__.py @@ -0,0 +1,52 @@ +"""Privacy filter for canonical telemetry (Issue 06). + +Public surface: + +* :mod:`research.telemetry.privacy.classify` - the field-classification + taxonomy and hard ``SECRET`` denylist. +* :mod:`research.telemetry.privacy.engine` - :class:`PrivacyPolicy`, + :func:`filter_event` and :func:`filter_payload`, which enforce data + minimization before any spool/log/retry/export. +""" + +from .classify import ( + BEHAVIORAL_TOKENS, + CODE_METADATA_TOKENS, + CONTENT_TOKENS, + SECRET_KEY_PATTERNS, + SECRET_VALUE_PATTERNS, + SYSTEM_TOKENS, + classify_event_payload, + classify_field, + contains_secret_value, + is_secret_key, + looks_secret_value, +) +from .engine import ( + REDACTED_MARKER, + FilterResult, + PrivacyPolicy, + filter_event, + filter_payload, + runtime_field_classes, +) + +__all__ = [ + "BEHAVIORAL_TOKENS", + "CODE_METADATA_TOKENS", + "CONTENT_TOKENS", + "FilterResult", + "PrivacyPolicy", + "REDACTED_MARKER", + "SECRET_KEY_PATTERNS", + "SECRET_VALUE_PATTERNS", + "SYSTEM_TOKENS", + "classify_event_payload", + "classify_field", + "contains_secret_value", + "filter_event", + "filter_payload", + "runtime_field_classes", + "is_secret_key", + "looks_secret_value", +] diff --git a/src/research/telemetry/privacy/classify.py b/src/research/telemetry/privacy/classify.py new file mode 100644 index 0000000..def5f96 --- /dev/null +++ b/src/research/telemetry/privacy/classify.py @@ -0,0 +1,286 @@ +"""Field classification for the canonical privacy filter. + +Classification is deterministic and fail-closed: + +1. A key matching the hard secret denylist, or a value matching a well-known + secret shape, is ``SECRET``. ``SECRET`` is never persistable regardless of + policy or consent. +2. Keys naming free content (prompts, reasoning, diffs, arguments, raw output) + are ``CONTENT``. +3. Keys naming file/code metadata are ``CODE_METADATA``. +4. Keys naming timing/process/schema facts are ``SYSTEM``. +5. Keys naming tool/permission/message lifecycle facts are ``BEHAVIORAL``. +6. Anything unrecognized is treated as ``CONTENT`` (fail closed): it is removed + unless content capture is explicitly allowed. Unrecognized *containers* are + traversed so their children can be classified individually. + +The classifier is vendor-independent and never invents a class from an ACP or +IDE method name. +""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from ..enums import FieldClass + +__all__ = [ + "CONTENT_TOKENS", + "CODE_METADATA_TOKENS", + "SECRET_KEY_PATTERNS", + "SECRET_VALUE_PATTERNS", + "SYSTEM_KEY_EXACT", + "SYSTEM_TOKENS", + "BEHAVIORAL_TOKENS", + "classify_event_payload", + "classify_field", + "contains_secret_value", + "is_secret_key", + "looks_secret_value", +] + +# Keys that are always secret, matched on a normalized (lowercased, +# underscored) key. ``tokens`` (a count) deliberately does NOT match ``token``. +SECRET_KEY_PATTERNS: tuple[str, ...] = ( + "password", + "passwd", + "secret", + "api_key", + "apikey", + "authorization", + "bearer", + "credential", + "private_key", + "access_key", + "client_secret", + "cookie", + "session_token", + "auth_token", +) +_SECRET_KEY_EXACT = frozenset( + {"token", "auth", "authorization", "secret", "password", "credential"} +) + +SECRET_VALUE_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile(r"\bsk-[A-Za-z0-9_-]{8,}"), + re.compile(r"\bghp_[A-Za-z0-9]{8,}"), + re.compile(r"\bAKIA[0-9A-Z]{12,}"), + re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]{4,}"), + re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"), +) + +CONTENT_TOKENS = frozenset( + { + "content", + "prompt", + "prompts", + "reasoning", + "thought", + "thoughts", + "completion", + "completions", + "response", + "output", + "stdout", + "stderr", + "diff", + "snippet", + "transcript", + "body", + "text", + "raw", + "payload", + "arguments", + "cot", + "chain", + "message_text", + } +) +CODE_METADATA_TOKENS = frozenset( + { + "path", + "file", + "filename", + "extension", + "language", + "line", + "lines", + "symbol", + "uri", + "directory", + "module", + "package", + "repo", + "workspace", + "document", + } +) +SYSTEM_TOKENS = frozenset( + { + "duration", + "latency", + "timestamp", + "clock", + "process", + "pid", + "version", + "schema", + "emitter", + "sequence", + "count", + "counts", + "size", + "bytes", + "host", + "os", + "arch", + "trace", + "span", + # Protocol metadata: the relay's ``streaming`` flag describes how the + # call was transported, not content. Without this the fail-closed + # default classifies it CONTENT and the whole relay fact is rejected. + "streaming", + "level", + "tokens", + "tokens_used", + } +) +# Exact normalized keys classified as ``SYSTEM``. ``exit_code`` is handled here +# rather than via a bare ``exit`` token: that token would also reclassify the +# proxy's ``exit_status`` payload key from BEHAVIORAL to SYSTEM. Mirrors the +# Kotlin FieldClassifier's ``systemKeyExact``. +SYSTEM_KEY_EXACT = frozenset( + { + "exit_code", + # Markers the server stamps on its own relay/self-report observations: + # which legacy fact a row is and the HTTP status of the server's own + # upstream call. They describe the relay's bookkeeping, never the + # participant; the dashboards read ``legacy_kind`` to find model and + # tool calls and reported zero under a METRICS-only policy while the + # ``kind``/``status`` tokens made them BEHAVIORAL. + "legacy_kind", + "upstream_status", + } +) +BEHAVIORAL_TOKENS = frozenset( + { + "tool", + "permission", + "turn", + "message", + "event", + "action", + "decision", + "status", + "state", + "role", + "model", + "result", + "outcome", + "error", + "name", + "id", + "type", + "call", + "edit", + "session", + "run", + "agent", + "capability", + "fidelity", + "kind", + "reason", + "method", + # ``phase`` -> BEHAVIORAL (parity with the Kotlin FieldClassifier). + "phase", + } +) + + +def _normalize_key(name: Any) -> str: + return re.sub(r"[^a-z0-9]+", "_", str(name).lower()).strip("_") + + +def _tokens(name: Any) -> set[str]: + return {token for token in re.split(r"[^a-z0-9]+", str(name).lower()) if token} + + +def is_secret_key(name: Any) -> bool: + """True when a mapping key names secret material.""" + normalized = _normalize_key(name) + if normalized in _SECRET_KEY_EXACT: + return True + if normalized.endswith(("_token", "_secret", "_password")): + return True + return any(pattern in normalized for pattern in SECRET_KEY_PATTERNS) + + +def looks_secret_value(value: Any) -> bool: + """True when a string value matches a well-known secret shape.""" + if not isinstance(value, str): + return False + return any(pattern.search(value) for pattern in SECRET_VALUE_PATTERNS) + + +def contains_secret_value(value: Any) -> bool: + """Recursively detect a secret-shaped value anywhere in ``value``.""" + if isinstance(value, Mapping): + return any( + is_secret_key(key) or contains_secret_value(item) + for key, item in value.items() + ) + if isinstance(value, (list, tuple)): + return any(contains_secret_value(item) for item in value) + return looks_secret_value(value) + + +def classify_field(name: Any, value: Any = None) -> FieldClass: + """Classify a single field by name (and value for secret shapes).""" + if is_secret_key(name) or looks_secret_value(value): + return FieldClass.SECRET + if _normalize_key(name) in SYSTEM_KEY_EXACT: + return FieldClass.SYSTEM + + # A bare magnitude of something ("tool_result_length": 42, "step_index": 0) + # is structural metadata, never content — even when the measured thing + # (arguments, results, ...) would itself be content. Without this, numeric + # telemetry like the relay's length-only tool metadata or step counters is + # rejected as CONTENT and takes the whole event down with it. + if isinstance(value, (int, float)) and not isinstance(value, bool): + suffixes = ("_length", "_count", "_counts", "_bytes", "_size", "_ms", "_index") + if _normalize_key(name).endswith(suffixes): + return FieldClass.SYSTEM + + tokens = _tokens(name) + if tokens & CONTENT_TOKENS: + return FieldClass.CONTENT + if tokens & CODE_METADATA_TOKENS: + return FieldClass.CODE_METADATA + if tokens & SYSTEM_TOKENS: + return FieldClass.SYSTEM + if tokens & BEHAVIORAL_TOKENS: + return FieldClass.BEHAVIORAL + + if isinstance(value, (Mapping, list, tuple)): + # Unrecognized containers are traversed; children are classified. + return FieldClass.SYSTEM + return FieldClass.CONTENT + + +def classify_event_payload(payload: Any, prefix: str = "") -> dict[str, FieldClass]: + """Flatten a sanitized payload into ``{dotted_path: FieldClass}``.""" + classified: dict[str, FieldClass] = {} + if isinstance(payload, Mapping): + for key, value in payload.items(): + path = f"{prefix}.{key}" if prefix else str(key) + classified[path] = classify_field(key, value) + if isinstance(value, (Mapping, list, tuple)): + classified.update(classify_event_payload(value, path)) + elif isinstance(payload, (list, tuple)): + for index, value in enumerate(payload): + path = f"{prefix}[{index}]" + classified[path] = classify_field(prefix, value) + if isinstance(value, (Mapping, list, tuple)): + classified.update(classify_event_payload(value, path)) + return classified diff --git a/src/research/telemetry/privacy/engine.py b/src/research/telemetry/privacy/engine.py new file mode 100644 index 0000000..5a06178 --- /dev/null +++ b/src/research/telemetry/privacy/engine.py @@ -0,0 +1,389 @@ +"""Shared privacy policy engine: classification, redaction and hashing. + +Redaction happens here, before any spool, log, retry queue, analytics, or +export. The engine is defensive in depth: even if a hand-crafted policy allows +everything, ``SECRET`` fields are dropped and a final recursive scan blocks any +residual secret shape from surviving serialization. + +Policy actions: + +* ``ALLOW`` - keep the value (``SYSTEM`` / ``BEHAVIORAL`` by default). +* ``REDACT`` - replace the value with a marker (``CONTENT`` without allowance). +* ``HASH`` - replace code metadata with a SHA-256 of its canonical JSON. +* ``DROP`` - remove the field entirely (``SECRET``). +* ``BLOCK`` - reject the whole event (explicitly blocked class). +""" + +from __future__ import annotations + +import re +from typing import Any, Mapping, Optional + +from pydantic import BaseModel, ConfigDict, Field + +from research.canonical import canonical_hash +from research.study.protocol.enums import TelemetryFieldClass + +from ..enums import FieldClass, PolicyAction +from ..models import CanonicalEventV1, PrivacySummary +from .classify import classify_field, contains_secret_value, looks_secret_value + +REDACTED_MARKER = "[REDACTED]" + +_BASE_CONFIG = ConfigDict(extra="forbid") + + +def _declared_field_classes(raw_classes: Any) -> set[FieldClass]: + """Map a study's ``allowed_field_classes`` onto runtime field classes. + + Studies are authored in the study vocabulary (STRUCTURAL / METRICS / + DIAGNOSTICS / CONTENT) and the study validator also accepts the runtime + names (SYSTEM / BEHAVIORAL / CODE_METADATA / CONTENT). Both are honoured, + with the same mapping ``from_revision_policy`` uses; unknown names are + ignored. Without this, a policy such as ``[STRUCTURAL, …, CONTENT]`` would + resolve to ``{CONTENT}`` alone and drop every structural field. + """ + classes: set[FieldClass] = set() + for raw in raw_classes if isinstance(raw_classes, (list, tuple, set)) else []: + value = raw.value if isinstance(raw, TelemetryFieldClass) else str(raw).upper() + if value == TelemetryFieldClass.STRUCTURAL.value: + classes.update({FieldClass.SYSTEM, FieldClass.BEHAVIORAL}) + elif value == TelemetryFieldClass.METRICS.value: + classes.add(FieldClass.SYSTEM) + elif value == TelemetryFieldClass.DIAGNOSTICS.value: + classes.add(FieldClass.BEHAVIORAL) + else: + try: + classes.add(FieldClass(value)) + except ValueError: + continue + return classes + + +def runtime_field_classes(raw_classes: Any) -> list[str]: + """The runtime class names a study's ``allowed_field_classes`` resolves to. + + Sorted and without ``SECRET``. An empty list means nothing usable was + declared, which every consumer reads as the metadata default (SYSTEM, + BEHAVIORAL, CODE_METADATA): ingestion here, the IDE plugin and the ACP + proxy it configures. The bootstrap manifest carries this list because the + plugin only understands the runtime vocabulary; given study names it + would keep only the runtime names it recognises and drop the rest. + """ + declared = _declared_field_classes(raw_classes) - {FieldClass.SECRET} + return sorted(field_class.value for field_class in declared) + + +class PrivacyPolicy(BaseModel): + """Resolved field policy for one revision + consent state. + + Derived from a revision ``TelemetryPolicy`` plus the participant's active + consent; ``SECRET`` is never in the allowed set and is enforced separately. + """ + + model_config = _BASE_CONFIG + + allowed_field_classes: list[FieldClass] = Field( + default_factory=lambda: [ + FieldClass.SYSTEM, + FieldClass.BEHAVIORAL, + FieldClass.CODE_METADATA, + ] + ) + # CONTENT is persisted only when explicitly allowed AND consent is active. + content_allowed: bool = False + consent_active: bool = False + policy_digest: Optional[str] = None + blocked_field_classes: list[FieldClass] = Field(default_factory=list) + # "hash" (default) or "allow" for CODE_METADATA. + code_metadata_mode: str = "hash" + + @classmethod + def default(cls) -> PrivacyPolicy: + """The deny-by-default policy: metadata only, no content capture.""" + return cls() + + @classmethod + def from_study_policy(cls, telemetry_policy: Mapping[str, Any], *, consent_active: bool) -> PrivacyPolicy: + """Resolve the current manifest policy using the IDE's field defaults. + + Accepts both the study-facing authoring vocabulary (STRUCTURAL / + METRICS / DIAGNOSTICS / CONTENT, the same names study creation + validates) and the runtime vocabulary (SYSTEM / BEHAVIORAL / + CODE_METADATA / CONTENT) — mirroring ``from_revision_policy``. An + authored allowlist the resolver doesn't understand must not silently + degrade to the default: without this, a researcher writing the + documented study vocabulary gets broader capture than they declared. + Content always needs the explicit content_capture flag and consent. + Code metadata is allowed when declared (or using the legacy/default + metadata policy); otherwise clients must upload SHA-256 tokens. + """ + # The same resolution the bootstrap manifest sends the IDE plugin + # (``runtime_field_classes``), so client and server agree. + declared = _declared_field_classes(telemetry_policy.get("allowed_field_classes") or []) - {FieldClass.SECRET} + allowed = declared or {FieldClass.SYSTEM, FieldClass.BEHAVIORAL, FieldClass.CODE_METADATA} + return cls( + allowed_field_classes=sorted(allowed, key=lambda item: item.value), + content_allowed=telemetry_policy.get("content_capture") is True, + consent_active=consent_active, + code_metadata_mode="allow" if FieldClass.CODE_METADATA in allowed else "hash", + policy_digest=canonical_hash(dict(telemetry_policy)), + ) + + @classmethod + def from_revision_policy( + cls, + telemetry_policy: Any, + *, + consent_active: bool, + code_metadata_mode: str = "hash", + ) -> PrivacyPolicy: + """Resolve a revision ``TelemetryPolicy`` + consent into a policy.""" + classes: set[FieldClass] = set() + content_allowed = False + raw_classes = getattr(telemetry_policy, "allowed_field_classes", []) or [] + for raw in raw_classes: + value = raw.value if isinstance(raw, TelemetryFieldClass) else str(raw).upper() + if value == TelemetryFieldClass.STRUCTURAL.value: + classes.update({FieldClass.SYSTEM, FieldClass.BEHAVIORAL}) + elif value == TelemetryFieldClass.METRICS.value: + classes.add(FieldClass.SYSTEM) + elif value == TelemetryFieldClass.DIAGNOSTICS.value: + classes.add(FieldClass.BEHAVIORAL) + elif value == TelemetryFieldClass.CONTENT.value: + content_allowed = True + classes.add(FieldClass.CONTENT) + else: + # Runtime vocabulary name (SYSTEM / BEHAVIORAL / CODE_METADATA / + # CONTENT): honour it exactly rather than silently dropping it. + try: + runtime_class = FieldClass(value) + except ValueError: + continue + if runtime_class is FieldClass.CONTENT: + content_allowed = True + classes.add(runtime_class) + + if not classes: + classes.update( + {FieldClass.SYSTEM, FieldClass.BEHAVIORAL, FieldClass.CODE_METADATA} + ) + + digest = canonical_hash( + { + "allowed": sorted(field_class.value for field_class in classes), + "content_allowed": content_allowed, + } + ) + return cls( + allowed_field_classes=sorted(classes, key=lambda item: item.value), + content_allowed=content_allowed, + consent_active=consent_active, + policy_digest=digest, + code_metadata_mode=code_metadata_mode, + ) + + def action_for(self, field_class: FieldClass) -> PolicyAction: + """Resolve the policy action for one field class.""" + if field_class in self.blocked_field_classes: + return PolicyAction.BLOCK + if field_class == FieldClass.SECRET: + return PolicyAction.DROP + if field_class == FieldClass.CONTENT: + if self.content_allowed and self.consent_active: + return PolicyAction.ALLOW + return PolicyAction.REDACT + if field_class == FieldClass.CODE_METADATA: + return ( + PolicyAction.HASH + if self.code_metadata_mode == "hash" + else PolicyAction.ALLOW + ) + if field_class in self.allowed_field_classes: + return PolicyAction.ALLOW + return PolicyAction.DROP + + +class FilterResult(BaseModel): + """A filtered event plus the summary of what the filter did.""" + + model_config = _BASE_CONFIG + + event: CanonicalEventV1 + summary: PrivacySummary + + +class _Blocked(Exception): + def __init__(self, path: str, field_class: FieldClass) -> None: + super().__init__(path) + self.path = path + self.field_class = field_class + + +_DROPPED = object() + + +def _hash_value(value: Any) -> str: + # Filtering at ingestion must preserve values already hashed by a producer. + if isinstance(value, str) and re.fullmatch(r"sha256:[0-9a-f]{64}", value): + return value + return "sha256:" + canonical_hash({"value": value}) + + +def _record( + actions: dict[str, int], classes: dict[str, int], action: PolicyAction, field_class: FieldClass +) -> None: + actions[action.value] = actions.get(action.value, 0) + 1 + classes[field_class.value] = classes.get(field_class.value, 0) + 1 + + +def _apply( + value: Any, + policy: PrivacyPolicy, + path: str, + actions: dict[str, int], + classes: dict[str, int], + redacted: list[str], +) -> Any: + if isinstance(value, Mapping): + filtered: dict[Any, Any] = {} + for key, item in value.items(): + child = f"{path}.{key}" if path else str(key) + field_class = classify_field(key, item) + action = policy.action_for(field_class) + _record(actions, classes, action, field_class) + if action == PolicyAction.BLOCK: + raise _Blocked(child, field_class) + if action == PolicyAction.DROP: + redacted.append(child) + continue + if action == PolicyAction.REDACT: + filtered[key] = REDACTED_MARKER + redacted.append(child) + continue + if action == PolicyAction.HASH: + filtered[key] = _hash_value(item) + redacted.append(child) + continue + filtered[key] = _apply(item, policy, child, actions, classes, redacted) + return filtered + + if isinstance(value, (list, tuple)): + items = [] + for index, item in enumerate(value): + child = f"{path}[{index}]" + result = _apply(item, policy, child, actions, classes, redacted) + if result is not _DROPPED: + items.append(result) + return items + + if looks_secret_value(value): + # Defense in depth for scalar list items with no classifying key. + _record(actions, classes, PolicyAction.DROP, FieldClass.SECRET) + redacted.append(path) + return _DROPPED + return value + + +def _auxiliary_secret_field(event: CanonicalEventV1) -> Optional[str]: + """Name of the first non-payload envelope field carrying a secret, or None. + + The privacy policy applies to the whole envelope: provenance, correlations, + lifecycle/unknown values and run identity are all scanned so a secret can + never ride along outside ``payload``. + """ + candidates: list[tuple[str, Any]] = [ + ("provenance", event.provenance.model_dump(mode="json")), + ("correlations", event.correlations.model_dump(mode="json")), + ("lifecycle_state", event.lifecycle_state), + ("unknown_event_type", event.unknown_event_type), + ("unknown_source", event.unknown_source), + ("unknown_lifecycle_state", event.unknown_lifecycle_state), + ("agent_run_id", event.agent_run_id), + ] + for name, value in candidates: + if value and contains_secret_value(value): + return name + return None + + +def filter_event(event: CanonicalEventV1, policy: PrivacyPolicy) -> FilterResult: + """Filter ``event`` under ``policy`` and return the event plus its summary.""" + actions: dict[str, int] = {} + classes: dict[str, int] = {} + redacted: list[str] = [] + blocked = False + block_reason: Optional[str] = None + + try: + filtered_payload: Any = _apply( + event.payload, policy, "", actions, classes, redacted + ) + except _Blocked as blocked_error: + blocked = True + block_reason = ( + f"field {blocked_error.path} classified " + f"{blocked_error.field_class.value} is blocked by policy" + ) + filtered_payload = {} + + if not blocked and contains_secret_value(filtered_payload): + blocked = True + block_reason = "residual secret detected after filtering" + filtered_payload = {} + + # Secrets are excluded from every envelope field, not only the payload: + # provenance, correlation ids, lifecycle/unknown values and run identity are + # scanned too and fail closed. A residual secret anywhere blocks the event. + if not blocked: + auxiliary = _auxiliary_secret_field(event) + if auxiliary is not None: + blocked = True + block_reason = f"residual secret detected in {auxiliary} after filtering" + filtered_payload = {} + + summary = PrivacySummary( + policy_digest=policy.policy_digest, + actions=actions, + redacted_fields=sorted(redacted), + blocked=blocked, + block_reason=block_reason, + field_classes=classes, + ) + filtered_event = event.model_copy( + update={"payload": dict(filtered_payload), "privacy": summary} + ) + return FilterResult(event=filtered_event, summary=summary) + + +def filter_payload( + payload: Mapping[str, Any], policy: PrivacyPolicy +) -> tuple[dict[str, Any], PrivacySummary]: + """Filter a bare payload mapping (convenience for tests and extractors).""" + actions: dict[str, int] = {} + classes: dict[str, int] = {} + redacted: list[str] = [] + blocked = False + block_reason: Optional[str] = None + try: + filtered: Any = _apply(payload, policy, "", actions, classes, redacted) + except _Blocked as blocked_error: + blocked = True + block_reason = ( + f"field {blocked_error.path} classified " + f"{blocked_error.field_class.value} is blocked by policy" + ) + filtered = {} + if not blocked and contains_secret_value(filtered): + blocked = True + block_reason = "residual secret detected after filtering" + filtered = {} + summary = PrivacySummary( + policy_digest=policy.policy_digest, + actions=actions, + redacted_fields=sorted(redacted), + blocked=blocked, + block_reason=block_reason, + field_classes=classes, + ) + return dict(filtered), summary diff --git a/src/research/telemetry/projections/__init__.py b/src/research/telemetry/projections/__init__.py new file mode 100644 index 0000000..4480021 --- /dev/null +++ b/src/research/telemetry/projections/__init__.py @@ -0,0 +1,38 @@ +"""Derived telemetry coverage and metric projections (Issue 09). + +Projections are versioned, derived views over immutable canonical events. They +never replace source events, and missing capability data is represented as +``value: null`` with an explicit coverage state rather than a zero measurement. +""" + +from .models import ( + COVERAGE_VERSION, + DERIVATION_VERSION, + DerivedMetricV1, + FamilyCoverage, + SessionRunSummaryV1, + TelemetryCoverageV1, +) +from .service import ( + as_canonical_event, + coverage_by_family, + derive_event_count_metric, + derive_usage_metric, + event_family, + session_summary, +) + +__all__ = [ + "COVERAGE_VERSION", + "DERIVATION_VERSION", + "DerivedMetricV1", + "FamilyCoverage", + "SessionRunSummaryV1", + "TelemetryCoverageV1", + "as_canonical_event", + "coverage_by_family", + "derive_event_count_metric", + "derive_usage_metric", + "event_family", + "session_summary", +] diff --git a/src/research/telemetry/projections/models.py b/src/research/telemetry/projections/models.py new file mode 100644 index 0000000..ef69d15 --- /dev/null +++ b/src/research/telemetry/projections/models.py @@ -0,0 +1,83 @@ +"""Pydantic v2 contracts for derived telemetry projections (Issue 09). + +Projections are derived, versioned views over immutable canonical events. A +missing measurement is represented as ``value: null`` plus an explicit coverage +state, never as a zero-valued measurement, and a projection never replaces the +source events. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - pydantic resolves model annotations at runtime +from typing import Optional +from uuid import UUID # noqa: TC003 - pydantic resolves model annotations at runtime + +from pydantic import BaseModel, ConfigDict, Field + +from research.telemetry.enums import CoverageState + +_FROZEN = ConfigDict(extra="forbid", frozen=True) +_BASE = ConfigDict(extra="forbid") + +COVERAGE_VERSION = "coverage-v1" +DERIVATION_VERSION = "derived-v1" + + +class FamilyCoverage(BaseModel): + """Explicit per-event-family coverage counts.""" + + model_config = _FROZEN + + family: str + total: int + available: int = 0 + unavailable: int = 0 + partial: int = 0 + unknown: int = 0 + needs_review: int = 0 + + +class TelemetryCoverageV1(BaseModel): + """Versioned coverage projection for one study and population.""" + + model_config = _FROZEN + + study_id: UUID + population: str + coverage_version: str = COVERAGE_VERSION + families: list[FamilyCoverage] = Field(default_factory=list) + denominators: dict[str, int] = Field(default_factory=dict) + computed_at: Optional[datetime] = None + + +class DerivedMetricV1(BaseModel): + """A versioned derived metric that preserves coverage semantics. + + ``value`` is ``None`` when the metric cannot be computed (no observed + denominator); that is a coverage statement, never a zero measurement. + """ + + model_config = _FROZEN + + metric_id: str + numerator: Optional[int] = None + denominator: Optional[int] = None + coverage_predicate: str + derivation_version: str = DERIVATION_VERSION + value: Optional[float] = None + coverage_state: CoverageState = CoverageState.UNKNOWN + source_event_count: int = 0 + + +class SessionRunSummaryV1(BaseModel): + """A per-session summary that retains source provenance.""" + + model_config = _FROZEN + + research_session_id: Optional[UUID] = None + event_count: int = 0 + family_counts: dict[str, int] = Field(default_factory=dict) + agent_run_ids: list[str] = Field(default_factory=list) + first_occurred_at: Optional[datetime] = None + last_occurred_at: Optional[datetime] = None + derivation_version: str = DERIVATION_VERSION diff --git a/src/research/telemetry/projections/service.py b/src/research/telemetry/projections/service.py new file mode 100644 index 0000000..4a13781 --- /dev/null +++ b/src/research/telemetry/projections/service.py @@ -0,0 +1,197 @@ +"""Pure projection functions over canonical research events (Issue 09). + +Every projection is deterministic, carries a derivation version and population, +and reads only committed event facts. Missing capability data stays visible as +``value: None`` with an explicit coverage state; projections never replace or +mutate the source events. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - used in annotations only +from typing import TYPE_CHECKING, Iterable, Optional, Sequence + +from research.telemetry.enums import CoverageState +from research.telemetry.models import CanonicalEventV1 + +from .models import ( + COVERAGE_VERSION, + DERIVATION_VERSION, + DerivedMetricV1, + FamilyCoverage, + SessionRunSummaryV1, + TelemetryCoverageV1, +) + +if TYPE_CHECKING: + from uuid import UUID + +__all__ = [ + "as_canonical_event", + "coverage_by_family", + "derive_event_count_metric", + "derive_usage_metric", + "event_family", + "session_summary", +] + + +def as_canonical_event(record: object) -> CanonicalEventV1: + """Return a canonical event for a stored record or a canonical event. + + The stored record carries the complete canonical envelope, so this is a + direct re-validation of the authority rather than a lossy field-by-field + reconstruction. + """ + if isinstance(record, CanonicalEventV1): + return record + envelope = getattr(record, "envelope", None) + if isinstance(envelope, dict) and envelope: + return CanonicalEventV1.model_validate(envelope) + if isinstance(record, dict): + return CanonicalEventV1.model_validate(record) + raise TypeError(f"cannot read a canonical event from {type(record)!r}") + + +def event_family(event_type: str) -> str: + """Return the canonical event family of a dotted event type.""" + return event_type.split(".", 1)[0] if event_type else "unknown" + + +def coverage_by_family( + events: Iterable[object], + study_id: UUID, + *, + population: str = "all_events", + computed_at: Optional[datetime] = None, +) -> TelemetryCoverageV1: + """Build the explicit per-family coverage projection for a study.""" + counts: dict[str, dict[str, int]] = {} + total = 0 + for record in events: + event = as_canonical_event(record) + total += 1 + family = event_family(event.event_type) + bucket = counts.setdefault( + family, + dict.fromkeys(CoverageState, 0), + ) + bucket[event.coverage.state.value] = bucket.get(event.coverage.state.value, 0) + 1 + + families = [ + FamilyCoverage( + family=family, + total=sum(bucket.values()), + available=bucket[CoverageState.AVAILABLE.value], + unavailable=bucket[CoverageState.UNAVAILABLE.value], + partial=bucket[CoverageState.PARTIAL.value], + unknown=bucket[CoverageState.UNKNOWN.value], + needs_review=bucket[CoverageState.NEEDS_REVIEW.value], + ) + for family, bucket in sorted(counts.items()) + ] + return TelemetryCoverageV1( + study_id=study_id, + population=population, + coverage_version=COVERAGE_VERSION, + families=families, + denominators={"events": total, "families": len(families)}, + computed_at=computed_at, + ) + + +def session_summary( + events: Iterable[object], + research_session_id: Optional[UUID] = None, +) -> SessionRunSummaryV1: + """Summarize the events of one session (or of all provided events).""" + selected = [ + as_canonical_event(record) + for record in events + if research_session_id is None + or record.research_session_id == research_session_id + ] + family_counts: dict[str, int] = {} + run_ids: set[str] = set() + first: Optional[datetime] = None + last: Optional[datetime] = None + for event in selected: + family = event_family(event.event_type) + family_counts[family] = family_counts.get(family, 0) + 1 + if event.agent_run_id: + run_ids.add(event.agent_run_id) + if first is None or event.occurred_at < first: + first = event.occurred_at + if last is None or event.occurred_at > last: + last = event.occurred_at + + return SessionRunSummaryV1( + research_session_id=research_session_id, + event_count=len(selected), + family_counts=family_counts, + agent_run_ids=sorted(run_ids), + first_occurred_at=first, + last_occurred_at=last, + derivation_version=DERIVATION_VERSION, + ) + + +def derive_usage_metric( + events: Sequence[object], + *, + metric_id: str = "usage_tokens_per_event", + population: str = "all_events", +) -> DerivedMetricV1: + """Derive mean usage tokens per event with explicit coverage. + + Only events whose ``usage_capability`` is ``AVAILABLE`` and whose value is + observed contribute; if none did, the value is ``None`` with ``UNAVAILABLE`` + coverage — never ``0``. + """ + canonical = [as_canonical_event(record) for record in events] + observed = [ + event + for event in canonical + if event.metrics.usage_capability.state == CoverageState.AVAILABLE + and event.metrics.usage_tokens is not None + ] + numerator = sum(event.metrics.usage_tokens or 0 for event in observed) + denominator = len(observed) + value: Optional[float] = None + state = CoverageState.UNAVAILABLE + if denominator > 0: + value = numerator / denominator + state = CoverageState.AVAILABLE + return DerivedMetricV1( + metric_id=metric_id, + numerator=numerator if denominator > 0 else None, + denominator=denominator if denominator > 0 else None, + coverage_predicate=( + f"{population}: usage_capability.state == AVAILABLE " + "and usage_tokens is not None" + ), + derivation_version=DERIVATION_VERSION, + value=value, + coverage_state=state, + source_event_count=len(canonical), + ) + + +def derive_event_count_metric( + events: Sequence[object], + *, + metric_id: str = "event_count", + population: str = "all_events", +) -> DerivedMetricV1: + """Derive the total event count (an observed count, not an inference).""" + count = len(events) + return DerivedMetricV1( + metric_id=metric_id, + numerator=count, + denominator=count, + coverage_predicate=f"{population}: all stored events", + derivation_version=DERIVATION_VERSION, + value=float(count), + coverage_state=CoverageState.AVAILABLE, + source_event_count=count, + ) diff --git a/src/research/telemetry/schema/__init__.py b/src/research/telemetry/schema/__init__.py new file mode 100644 index 0000000..f290d41 --- /dev/null +++ b/src/research/telemetry/schema/__init__.py @@ -0,0 +1,95 @@ +"""Language-neutral JSON Schema for the canonical telemetry envelope. + +``canonical_event_v1.json`` is **generated** from +:class:`research.telemetry.models.CanonicalEventV1`, which is the single source +of truth for the event schema; the closed vocabularies (event types, coverage +states, fidelity) come from :mod:`research.telemetry.enums`. It is the +cross-language contract consumed by generated clients. + +The generation is deterministic and is asserted by the test suite, so the +checked-in file can never silently drift from the Python model:: + + python scripts/dev/generate_canonical_event_schema.py + +Unknown ``source``/``lifecycle`` semantics are identical to the model: both +accept an unknown token verbatim (the original is preserved on +``unknown_source``/``unknown_lifecycle_state``), so the schema does not enum- +restrict them. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from research.telemetry.enums import CANONICAL_SCHEMA_VERSION, CanonicalEventType +from research.telemetry.models import CanonicalEventV1 + +SCHEMA_DIR = Path(__file__).resolve().parent +CANONICAL_EVENT_V1_SCHEMA_PATH = SCHEMA_DIR / "canonical_event_v1.json" + +_SCHEMA_ID = "https://code4me.invalid/research/schema/canonical_event_v1.json" +_SCHEMA_DESCRIPTION = ( + "Language-neutral canonical telemetry envelope (schema version " + f"{CANONICAL_SCHEMA_VERSION}). Canonical event types are vendor-independent " + "concepts; SECRET fields never appear because the privacy filter removes " + "them before serialization." +) + + +def build_canonical_event_v1_schema() -> dict[str, Any]: + """Build the canonical event JSON Schema from the Python model.""" + schema = CanonicalEventV1.model_json_schema(ref_template="#/$defs/{model}") + schema["$schema"] = "https://json-schema.org/draft/2020-12/schema" + schema["$id"] = _SCHEMA_ID + schema["title"] = "CanonicalEventV1" + schema["description"] = _SCHEMA_DESCRIPTION + + properties = schema.setdefault("properties", {}) + # The envelope version is required and pinned; it is never defaulted. + properties["schema_version"] = {"const": CANONICAL_SCHEMA_VERSION} + # Per-emitter sequence is a strictly positive monotonic counter. + properties["emitter_sequence"] = { + **properties["emitter_sequence"], + "minimum": 1, + } + # The builder guarantees a known canonical type (an unrecognized source + # construct becomes ``unknown_source_event``), so the wire vocabulary is + # closed even though the raw model field is a plain string. + properties["event_type"] = { + "type": "string", + "enum": [event_type.value for event_type in CanonicalEventType], + } + return schema + + +def serialize_canonical_event_v1_schema(schema: dict[str, Any]) -> str: + """Serialize the schema deterministically (stable for review and diffs).""" + return json.dumps(schema, indent=2, ensure_ascii=False) + "\n" + + +def write_canonical_event_v1_schema( + path: Path = CANONICAL_EVENT_V1_SCHEMA_PATH, +) -> Path: + """Regenerate the checked-in JSON Schema from the Python model.""" + path.write_text( + serialize_canonical_event_v1_schema(build_canonical_event_v1_schema()), + encoding="utf-8", + ) + return path + + +def load_canonical_event_v1_schema() -> dict[str, Any]: + """Load the canonical event JSON Schema.""" + return json.loads(CANONICAL_EVENT_V1_SCHEMA_PATH.read_text()) + + +__all__ = [ + "CANONICAL_EVENT_V1_SCHEMA_PATH", + "SCHEMA_DIR", + "build_canonical_event_v1_schema", + "load_canonical_event_v1_schema", + "serialize_canonical_event_v1_schema", + "write_canonical_event_v1_schema", +] diff --git a/src/research/telemetry/schema/canonical_event_v1.json b/src/research/telemetry/schema/canonical_event_v1.json new file mode 100644 index 0000000..ea553d6 --- /dev/null +++ b/src/research/telemetry/schema/canonical_event_v1.json @@ -0,0 +1,553 @@ +{ + "$defs": { + "CanonicalFidelity": { + "description": "How faithfully a canonical field represents its source observation.\n\nLowercase by canonical-spec convention; distinct from the uppercase receipt\nfidelity vocabulary in Issue 01.", + "enum": [ + "exact", + "normalized", + "inferred" + ], + "title": "CanonicalFidelity", + "type": "string" + }, + "Correlations": { + "additionalProperties": false, + "description": "Opaque correlation handles that tie related events together.", + "properties": { + "turn_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Turn Id" + }, + "tool_call_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Call Id" + }, + "permission_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Permission Id" + }, + "edit_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Edit Id" + }, + "correlation_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Correlation Id" + }, + "message_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Message Id" + }, + "trace_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Trace Id" + }, + "span_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Span Id" + }, + "parent_span_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Parent Span Id" + }, + "model_call_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Call Id" + } + }, + "title": "Correlations", + "type": "object" + }, + "Coverage": { + "additionalProperties": false, + "description": "Coverage of one measured/observed field or capability.", + "properties": { + "state": { + "$ref": "#/$defs/CoverageState", + "default": "UNKNOWN" + }, + "reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Reason" + }, + "capability": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Capability" + } + }, + "title": "Coverage", + "type": "object" + }, + "CoverageState": { + "description": "Whether a measurement/observation is present, absent, or disputed.\n\n``UNAVAILABLE`` means \"the source does not expose it\" and ``UNKNOWN`` means\n\"not observed\"; neither is zero or false.", + "enum": [ + "AVAILABLE", + "UNAVAILABLE", + "UNKNOWN", + "PARTIAL", + "NEEDS_REVIEW" + ], + "title": "CoverageState", + "type": "string" + }, + "EventMetrics": { + "additionalProperties": false, + "description": "Numeric measurements attached to an event.\n\n``usage_tokens`` is ``Optional`` and paired with ``usage_capability``:\nan observed zero is ``0`` with ``AVAILABLE`` coverage, while an unexposed\ncount is ``None`` with ``UNAVAILABLE`` coverage.", + "properties": { + "usage_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Usage Tokens" + }, + "usage_capability": { + "$ref": "#/$defs/Coverage" + }, + "latency_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Latency Ms" + }, + "counts": { + "additionalProperties": { + "type": "integer" + }, + "title": "Counts", + "type": "object" + } + }, + "title": "EventMetrics", + "type": "object" + }, + "PrivacySummary": { + "additionalProperties": false, + "description": "What the privacy filter did to one event.", + "properties": { + "policy_digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Policy Digest" + }, + "actions": { + "additionalProperties": { + "type": "integer" + }, + "title": "Actions", + "type": "object" + }, + "redacted_fields": { + "items": { + "type": "string" + }, + "title": "Redacted Fields", + "type": "array" + }, + "blocked": { + "default": false, + "title": "Blocked", + "type": "boolean" + }, + "block_reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Block Reason" + }, + "field_classes": { + "additionalProperties": { + "type": "integer" + }, + "title": "Field Classes", + "type": "object" + } + }, + "title": "PrivacySummary", + "type": "object" + }, + "Provenance": { + "additionalProperties": false, + "description": "Where a canonical event came from and how faithfully it was derived.", + "properties": { + "source": { + "title": "Source", + "type": "string" + }, + "source_event_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Event Id" + }, + "normalizer_version": { + "title": "Normalizer Version", + "type": "string" + }, + "adapter_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Adapter Version" + }, + "fidelity": { + "$ref": "#/$defs/CanonicalFidelity", + "default": "normalized" + }, + "evidence_digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Evidence Digest" + } + }, + "required": [ + "source", + "normalizer_version" + ], + "title": "Provenance", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Language-neutral canonical telemetry envelope (schema version 1). Canonical event types are vendor-independent concepts; SECRET fields never appear because the privacy filter removes them before serialization.", + "properties": { + "event_id": { + "format": "uuid", + "title": "Event Id", + "type": "string" + }, + "schema_version": { + "const": "1" + }, + "event_type": { + "type": "string", + "enum": [ + "interaction.started", + "interaction.completed", + "agent.run.started", + "agent.run.completed", + "agent.message.started", + "agent.message.completed", + "tool.created", + "tool.started", + "tool.completed", + "tool.failed", + "permission.requested", + "permission.decided", + "plan.updated", + "usage.updated", + "ide.document.changed", + "ide.file.opened", + "ide.file.saved", + "ide.file.closed", + "ide.run.executed", + "system.agent.crashed", + "system.proxy.error", + "agent.error", + "unknown_source_event" + ] + }, + "source": { + "title": "Source", + "type": "string" + }, + "study_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Study Id" + }, + "enrollment_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Enrollment Id" + }, + "research_session_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Research Session Id" + }, + "agent_run_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Agent Run Id" + }, + "occurred_at": { + "format": "date-time", + "title": "Occurred At", + "type": "string" + }, + "monotonic_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Monotonic Ns" + }, + "emitter_id": { + "title": "Emitter Id", + "type": "string" + }, + "emitter_sequence": { + "title": "Emitter Sequence", + "type": "integer", + "minimum": 1 + }, + "correlations": { + "$ref": "#/$defs/Correlations" + }, + "lifecycle_state": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Lifecycle State" + }, + "payload": { + "additionalProperties": true, + "title": "Payload", + "type": "object" + }, + "metrics": { + "$ref": "#/$defs/EventMetrics" + }, + "privacy": { + "$ref": "#/$defs/PrivacySummary" + }, + "provenance": { + "$ref": "#/$defs/Provenance" + }, + "coverage": { + "$ref": "#/$defs/Coverage" + }, + "unknown_event_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Unknown Event Type" + }, + "unknown_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Unknown Source" + }, + "unknown_lifecycle_state": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Unknown Lifecycle State" + } + }, + "required": [ + "event_id", + "schema_version", + "event_type", + "source", + "occurred_at", + "emitter_id", + "emitter_sequence", + "provenance" + ], + "title": "CanonicalEventV1", + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://code4me.invalid/research/schema/canonical_event_v1.json" +} diff --git a/src/research/telemetry/validation.py b/src/research/telemetry/validation.py new file mode 100644 index 0000000..a263f7c --- /dev/null +++ b/src/research/telemetry/validation.py @@ -0,0 +1,239 @@ +"""Typed validation for canonical telemetry envelopes. + +Validation never raises for data: it returns a list of typed +:class:`TelemetryValidationError`. Unknown-but-preserved values produce a +``NEEDS_REVIEW`` finding rather than a hard rejection, while structural +problems (missing id, missing provenance, bad sequence) are errors. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Any, Mapping, Union + +from pydantic import BaseModel, ConfigDict +from pydantic import ValidationError as PydanticValidationError + +from .enums import CanonicalEventType, EventSource +from .models import CanonicalEventV1 +from .privacy.classify import contains_secret_value + +__all__ = [ + "TelemetryValidationCode", + "TelemetryValidationError", + "validate_canonical_event", +] + +_SCHEMA_VERSION = "1" +_KNOWN_EVENT_TYPES = frozenset(event_type.value for event_type in CanonicalEventType) +_KNOWN_SOURCES = frozenset(source.value for source in EventSource) + + +class TelemetryValidationCode(str, Enum): + """Stable, machine-readable canonical-event validation reason codes.""" + + SCHEMA_INVALID = "SCHEMA_INVALID" + SCHEMA_VERSION_UNSUPPORTED = "SCHEMA_VERSION_UNSUPPORTED" + SCHEMA_VERSION_MISSING = "SCHEMA_VERSION_MISSING" + MISSING_EVENT_ID = "MISSING_EVENT_ID" + MISSING_PROVENANCE = "MISSING_PROVENANCE" + MISSING_EMITTER_ID = "MISSING_EMITTER_ID" + INVALID_EMITTER_SEQUENCE = "INVALID_EMITTER_SEQUENCE" + MISSING_OCCURRED_AT = "MISSING_OCCURRED_AT" + UNKNOWN_EVENT_TYPE = "UNKNOWN_EVENT_TYPE" + UNKNOWN_SOURCE = "UNKNOWN_SOURCE" + SECRET_PRESENT = "SECRET_PRESENT" + PRIVACY_BLOCKED = "PRIVACY_BLOCKED" + + +class TelemetryValidationError(BaseModel): + """One typed validation finding.""" + + model_config = ConfigDict(extra="forbid") + + code: TelemetryValidationCode + field: str + message: str + needs_review: bool = False + + +def _issue( + code: TelemetryValidationCode, field: str, message: str, *, needs_review: bool = False +) -> TelemetryValidationError: + return TelemetryValidationError( + code=code, field=field, message=message, needs_review=needs_review + ) + + +def _validate_model(event: CanonicalEventV1) -> list[TelemetryValidationError]: + errors: list[TelemetryValidationError] = [] + + if event.schema_version != _SCHEMA_VERSION: + errors.append( + _issue( + TelemetryValidationCode.SCHEMA_VERSION_UNSUPPORTED, + "schema_version", + f"unsupported schema_version {event.schema_version!r}", + ) + ) + + if event.event_id is None: + errors.append( + _issue( + TelemetryValidationCode.MISSING_EVENT_ID, "event_id", "event_id is required" + ) + ) + + if event.provenance is None: + errors.append( + _issue( + TelemetryValidationCode.MISSING_PROVENANCE, + "provenance", + "provenance is required", + ) + ) + + if not event.emitter_id or not str(event.emitter_id).strip(): + errors.append( + _issue( + TelemetryValidationCode.MISSING_EMITTER_ID, + "emitter_id", + "emitter_id is required for per-emitter ordering", + ) + ) + + if event.emitter_sequence is None or event.emitter_sequence < 1: + errors.append( + _issue( + TelemetryValidationCode.INVALID_EMITTER_SEQUENCE, + "emitter_sequence", + "emitter_sequence must be a strictly positive integer", + ) + ) + + if event.occurred_at is None: + errors.append( + _issue( + TelemetryValidationCode.MISSING_OCCURRED_AT, + "occurred_at", + "occurred_at is required", + ) + ) + + if event.privacy.blocked: + errors.append( + _issue( + TelemetryValidationCode.PRIVACY_BLOCKED, + "privacy.blocked", + "the privacy filter blocked this event; a blocked event is never persistable", + ) + ) + + if event.event_type not in _KNOWN_EVENT_TYPES: + errors.append( + _issue( + TelemetryValidationCode.UNKNOWN_EVENT_TYPE, + "event_type", + f"unrecognized event_type {event.event_type!r}", + needs_review=True, + ) + ) + elif event.unknown_event_type is not None: + errors.append( + _issue( + TelemetryValidationCode.UNKNOWN_EVENT_TYPE, + "unknown_event_type", + f"source event type {event.unknown_event_type!r} was preserved", + needs_review=True, + ) + ) + + if event.source not in _KNOWN_SOURCES: + errors.append( + _issue( + TelemetryValidationCode.UNKNOWN_SOURCE, + "source", + f"unrecognized source {event.source!r}", + needs_review=True, + ) + ) + elif event.unknown_source is not None: + errors.append( + _issue( + TelemetryValidationCode.UNKNOWN_SOURCE, + "unknown_source", + f"source {event.unknown_source!r} was preserved", + needs_review=True, + ) + ) + + # Defense in depth: a filtered event must not still carry secret-shaped + # material anywhere it can be persisted. Provenance and correlation handles + # are attacker-influenced wire fields, so they are scanned too (not only the + # payload/metrics). + for field_name, value in ( + ("payload", event.payload), + ("metrics", event.metrics.model_dump(mode="json")), + ("provenance", event.provenance.model_dump(mode="json")), + ("correlations", event.correlations.model_dump(mode="json")), + ("lifecycle_state", event.lifecycle_state), + ("unknown_event_type", event.unknown_event_type), + ("unknown_source", event.unknown_source), + ("unknown_lifecycle_state", event.unknown_lifecycle_state), + ): + if contains_secret_value(value): + errors.append( + _issue( + TelemetryValidationCode.SECRET_PRESENT, + field_name, + f"{field_name} contains secret material and must be filtered first", + ) + ) + + return errors + + +def validate_canonical_event( + event_or_dict: Union[CanonicalEventV1, Mapping[str, Any]], +) -> list[TelemetryValidationError]: + """Return every typed validation finding for a canonical event.""" + if isinstance(event_or_dict, CanonicalEventV1): + return _validate_model(event_or_dict) + + if not isinstance(event_or_dict, Mapping): + return [ + _issue( + TelemetryValidationCode.SCHEMA_INVALID, + "", + "canonical event must be a CanonicalEventV1 or a mapping", + ) + ] + + try: + event = CanonicalEventV1.model_validate(dict(event_or_dict)) + except PydanticValidationError as error: + details = error.errors() + # A missing schema_version is its own typed reason: the envelope version + # is never defaulted, so "absent" must be distinguishable from + # "present but unsupported". + for detail in details: + location = tuple(detail.get("loc", ())) + if detail.get("type") == "missing" and location == ("schema_version",): + return [ + _issue( + TelemetryValidationCode.SCHEMA_VERSION_MISSING, + "schema_version", + "schema_version is required; it is never defaulted on ingestion", + ) + ] + first = details[0] if details else {} + field = ".".join(str(part) for part in first.get("loc", ())) + return [ + _issue( + TelemetryValidationCode.SCHEMA_INVALID, + field, + str(first.get("msg", "canonical event failed schema validation")), + ) + ] + + return _validate_model(event) diff --git a/src/response_models.py b/src/response_models.py index abb760c..113755c 100644 --- a/src/response_models.py +++ b/src/response_models.py @@ -36,6 +36,12 @@ class ResponseUser(Queries.CreateUser): default=None, description="Last authentication token used by the user" ) is_admin: bool = Field(False, description="Whether the user is an admin") + # Administrator-enabled researcher account (P3): the single researcher + # authority. Exposed so the website can gate researcher surfaces without a + # per-study role lookup. + can_research: bool = Field( + False, description="Whether an admin enabled this account for research" + ) @field_validator("preference", mode="before") @classmethod diff --git a/src/website/Dockerfile b/src/website/Dockerfile new file mode 100644 index 0000000..11327e8 --- /dev/null +++ b/src/website/Dockerfile @@ -0,0 +1,30 @@ +# Production website image: static React build served by nginx, which also +# proxies /api/ to the backend (production-readiness B-07). Build context is the +# repository root (docker-compose.prod.yml) so nginx.prod.conf can be baked in. +# +# The backend origin is compiled into the bundle by Create React App, so it is a +# build argument: pass the public origin the participants use (same origin as +# the website, since nginx proxies /api/ there). + +FROM node:22-alpine AS build +WORKDIR /website +# .npmrc carries legacy-peer-deps=true: the lock file was generated with it, so +# `npm ci` must see it too or it refuses the lock as out of sync. +COPY src/website/package.json src/website/package-lock.json src/website/.npmrc ./ +RUN npm ci --no-audit --no-fund +COPY src/website/ ./ +ARG REACT_APP_BACKEND_HOST +ARG REACT_APP_BACKEND_PORT= +ENV REACT_APP_BACKEND_HOST=$REACT_APP_BACKEND_HOST \ + REACT_APP_BACKEND_PORT=$REACT_APP_BACKEND_PORT \ + GENERATE_SOURCEMAP=false \ + CI=true +RUN test -n "$REACT_APP_BACKEND_HOST" || (echo "REACT_APP_BACKEND_HOST build argument is required" && exit 1) +RUN npm run build + +FROM nginx:1.27.5-alpine +COPY nginx.prod.conf /etc/nginx/nginx.conf +COPY --from=build /website/build /usr/share/nginx/html +EXPOSE 80 +HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ + CMD wget -q --spider http://127.0.0.1/nginx-health || exit 1 diff --git a/src/website/package-lock.json b/src/website/package-lock.json index 033ae67..7a21d20 100644 --- a/src/website/package-lock.json +++ b/src/website/package-lock.json @@ -8,7 +8,6 @@ "name": "website", "version": "0.1.0", "dependencies": { - "@react-oauth/google": "^0.12.1", "@testing-library/dom": "^10.4.0", "@testing-library/jest-dom": "^6.6.3", "@testing-library/react": "^16.3.0", @@ -16,7 +15,6 @@ "crypto-js": "^4.2.0", "dotenv": "^16.5.0", "hocon-parser": "^1.0.0", - "jwt-decode": "^4.0.0", "react": "^19.1.0", "react-dom": "^19.1.0", "react-router-dom": "^6.26.2", @@ -2894,15 +2892,6 @@ } } }, - "node_modules/@react-oauth/google": { - "version": "0.12.1", - "resolved": "https://registry.npmjs.org/@react-oauth/google/-/google-0.12.1.tgz", - "integrity": "sha512-qagsy22t+7UdkYAiT5ZhfM4StXi9PPNvw0zuwNmabrWyMKddczMtBIOARflbaIj+wHiQjnMAsZmzsUYuXeyoSg==", - "peerDependencies": { - "react": ">=16.8.0", - "react-dom": ">=16.8.0" - } - }, "node_modules/@remix-run/router": { "version": "1.23.0", "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.0.tgz", @@ -10235,14 +10224,6 @@ "node": ">=4.0" } }, - "node_modules/jwt-decode": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-4.0.0.tgz", - "integrity": "sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==", - "engines": { - "node": ">=18" - } - }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", diff --git a/src/website/package.json b/src/website/package.json index 6b7aa0a..3e811f8 100644 --- a/src/website/package.json +++ b/src/website/package.json @@ -3,14 +3,12 @@ "version": "0.1.0", "private": true, "dependencies": { - "@react-oauth/google": "^0.12.1", "@testing-library/dom": "^10.4.0", "@testing-library/jest-dom": "^6.6.3", "@testing-library/react": "^16.3.0", "@testing-library/user-event": "^13.5.0", "crypto-js": "^4.2.0", "dotenv": "^16.5.0", - "jwt-decode": "^4.0.0", "react": "^19.1.0", "react-dom": "^19.1.0", "react-router-dom": "^6.26.2", diff --git a/src/website/src/App.css b/src/website/src/App.css index 37d70f0..88adff5 100644 --- a/src/website/src/App.css +++ b/src/website/src/App.css @@ -1,42 +1,108 @@ /* App.css - Main application styles */ -/* Theme variables */ +/* Theme variables. The original names stay the public contract for older + views; the semantic tokens below them drive the shared UI kit (styles/ui.css). */ :root { /* Light theme (default) */ --primary-color: #4285f4; - --primary-hover: #3367d6; - --secondary-color: #f5f5f5; + --primary-hover: #2f6fd8; + --primary-contrast: #ffffff; + --primary-soft: rgba(66, 133, 244, 0.1); + --secondary-color: #f1f3f6; --background-color: #f5f7fa; --card-background: #ffffff; - --text-color: #333333; - --text-secondary: #666666; - --border-color: #dddddd; - --error-color: #d32f2f; - --success-color: #2e7d32; - --info-color: #1976d2; - --hover-color: rgba(0, 0, 0, 0.05); - --focus-color: rgba(66, 133, 244, 0.4); - --shadow-color: rgba(0, 0, 0, 0.1); + --surface-muted: #f8f9fb; + --surface-sunken: #eef1f5; + --text-color: #1f2733; + --text-secondary: #5b6573; + --text-muted: #8a939f; + --border-color: #e1e5eb; + --border-strong: #cfd5dd; + --error-color: #c62828; + --success-color: #1e7b34; + --info-color: #1767c6; + --warning-color: #9a5b00; + --danger-soft: rgba(198, 40, 40, 0.08); + --success-soft: rgba(30, 123, 52, 0.09); + --warning-soft: rgba(214, 138, 0, 0.12); + --info-soft: rgba(23, 103, 198, 0.08); + --violet-color: #5b45c2; + --violet-soft: rgba(91, 69, 194, 0.1); + --hover-color: rgba(15, 23, 42, 0.04); + --focus-color: rgba(66, 133, 244, 0.35); + --shadow-color: rgba(15, 23, 42, 0.08); + --shadow-sm: 0 1px 2px rgba(15, 23, 42, 0.05); + --shadow-md: 0 4px 16px rgba(15, 23, 42, 0.08); + --shadow-lg: 0 12px 40px rgba(15, 23, 42, 0.16); + --radius-sm: 6px; + --radius-md: 8px; + --radius-lg: 12px; --chart-grid: rgba(0, 0, 0, 0.05); + /* Chart palette (validated with the dataviz six-checks script against + #ffffff: all hard gates pass; slots 3-5 need visible labels/table view). */ + --viz-1: #2a78d6; + --viz-2: #eb6834; + --viz-3: #1baf7a; + --viz-4: #eda100; + --viz-5: #e87ba4; + --viz-6: #008300; + --viz-7: #4a3aa7; + --viz-8: #e34948; + --viz-muted: #c3c7ce; + --viz-grid: #e8ebef; + --viz-axis: #c9ced6; + --status-good: #0ca30c; + --status-warning: #fab219; + --status-serious: #ec835a; + --status-critical: #d03b3b; } /* Dark theme */ [data-theme="dark"] { --primary-color: #5c9aff; --primary-hover: #7eadff; - --secondary-color: #333333; - --background-color: #121212; - --card-background: #1e1e1e; - --text-color: #f5f5f5; - --text-secondary: #bbbbbb; - --border-color: #444444; - --error-color: #f44336; - --success-color: #4caf50; - --info-color: #2196f3; - --hover-color: rgba(255, 255, 255, 0.1); - --focus-color: rgba(92, 154, 255, 0.4); - --shadow-color: rgba(0, 0, 0, 0.3); + --primary-contrast: #0b1220; + --primary-soft: rgba(92, 154, 255, 0.14); + --secondary-color: #262a31; + --background-color: #111317; + --card-background: #1a1d23; + --surface-muted: #1f2329; + --surface-sunken: #15181d; + --text-color: #eceef1; + --text-secondary: #a9b1bc; + --text-muted: #7c8591; + --border-color: #2d323b; + --border-strong: #3a414c; + --error-color: #ff6b6b; + --success-color: #4cc26a; + --info-color: #6aa8ff; + --warning-color: #f2b64a; + --danger-soft: rgba(255, 107, 107, 0.12); + --success-soft: rgba(76, 194, 106, 0.13); + --warning-soft: rgba(242, 182, 74, 0.13); + --info-soft: rgba(106, 168, 255, 0.13); + --violet-color: #a99cf5; + --violet-soft: rgba(169, 156, 245, 0.14); + --hover-color: rgba(255, 255, 255, 0.06); + --focus-color: rgba(92, 154, 255, 0.45); + --shadow-color: rgba(0, 0, 0, 0.35); + --shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.3); + --shadow-md: 0 4px 16px rgba(0, 0, 0, 0.35); + --shadow-lg: 0 12px 40px rgba(0, 0, 0, 0.5); --chart-grid: rgba(255, 255, 255, 0.1); + /* Dark steps of the same eight hues, validated against the dark card + surface #1a1d23: all checks pass, every slot >= 3:1. */ + --viz-1: #3987e5; + --viz-2: #d95926; + --viz-3: #199e70; + --viz-4: #c98500; + --viz-5: #d55181; + --viz-6: #008300; + --viz-7: #9085e9; + --viz-8: #e66767; + --viz-muted: #4a515c; + --viz-grid: #2a2f37; + --viz-axis: #3a414c; } * { diff --git a/src/website/src/App.js b/src/website/src/App.js index 117e42c..b8526d6 100644 --- a/src/website/src/App.js +++ b/src/website/src/App.js @@ -1,149 +1,225 @@ -import React, { useState, useEffect } from "react"; +import React, { startTransition, useState, useEffect } from "react"; import Auth from "./components/auth/Auth"; import Dashboard from "./pages/Dashboard"; +import ResearchStudies from "./pages/research/ResearchStudies"; +import ResearchStudyEditor from "./pages/research/ResearchStudyEditor"; +import ResearchJoin from "./pages/research/ResearchJoin"; +import PrivacySettings from "./pages/PrivacySettings"; +import AppShell, { homePath } from "./components/layout/AppShell"; import "./App.css"; -import { GoogleOAuthProvider } from "@react-oauth/google"; import { ThemeProvider } from "./context/ThemeContext"; import { getCurrentUser, logoutUser } from "./utils/api"; -import { BrowserRouter, Routes, Route, Navigate, useNavigate } from "react-router-dom"; +import { + BrowserRouter, + Routes, + Route, + Navigate, + Outlet, + useNavigate, +} from "react-router-dom"; import Start from "./pages/Start"; + +const USER_STORAGE_KEY = "user"; +const RESEARCH_JOIN_INTENT_KEY = "code4me.research.join.intent"; + +// The session lives in the httpOnly auth cookie. Only non-secret profile +// fields are cached locally: the login response also carries the live auth +// token (and a masked password), which must never be copied into +// localStorage where any script on the origin could read it. +// React Router 6.26 warns about its v7 behaviours until they are opted in. +const ROUTER_FUTURE = { v7_startTransition: true, v7_relativeSplatPath: true }; + +const PUBLIC_USER_FIELDS = ["user_id", "email", "name", "is_admin", "can_research", "verified", "joined_at"]; + +export const sanitizeUser = (user) => { + if (!user || typeof user !== "object") return null; + return PUBLIC_USER_FIELDS.reduce((safe, key) => { + if (user[key] !== undefined) safe[key] = user[key]; + return safe; + }, {}); +}; + +const rememberUser = (user) => { + try { + const safe = sanitizeUser(user); + if (safe) localStorage.setItem(USER_STORAGE_KEY, JSON.stringify({ user: safe })); + else localStorage.removeItem(USER_STORAGE_KEY); + } catch (_) { + // Storage can be unavailable (private mode); the cookie session still works. + } +}; + +const forgetUser = () => { + try { + localStorage.removeItem(USER_STORAGE_KEY); + } catch (_) { + // ignore + } +}; + function App() { - // we manage the state for the uer but setting it to null by default const [user, setUser] = useState(null); const [isLoading, setIsLoading] = useState(true); - // useEffect to check if the user is already logged in via auth token cookie + // Resolve the session from the server-side auth cookie on load. useEffect(() => { const checkAuthStatus = async () => { try { - // First check if we have a valid server session const response = await getCurrentUser(); if (response.ok) { setUser(response.user); - // Store user data in localStorage for quick access - localStorage.setItem("user", JSON.stringify({ - user: response.user, - config: response.config - })); + rememberUser(response.user); } else { - // Server says no valid session, clear everything setUser(null); - localStorage.removeItem("user"); + forgetUser(); } } catch (error) { console.error("Error checking auth status:", error); - // Network error or server issue, clear local state to be safe setUser(null); - localStorage.removeItem("user"); + forgetUser(); } finally { setIsLoading(false); } }; - - // Always check server-side auth status first checkAuthStatus(); }, []); - // handleAuthenticated function to set the user state const handleAuthenticated = (userData) => { setUser(userData.user); - localStorage.setItem("user", JSON.stringify(userData)); + rememberUser(userData.user); + }; + + // The local half of a logout, also used after account deletion (where the + // server has already ended the session). + const clearLocalSession = () => { + setUser(null); + forgetUser(); + sessionStorage.clear(); }; - // handleLogout function to properly clear user session const handleLogout = async () => { - console.log("Logging out user..."); - try { - // Call the logout API - this will clear server-side session and cookies const logoutResponse = await logoutUser(); - console.log("Logout API response:", logoutResponse); - if (!logoutResponse.ok) { console.warn("Logout API failed, but continuing with local cleanup"); } } catch (error) { console.error("Error during logout API call:", error); - console.log("Continuing with local cleanup despite API error"); } - - // Clear local state (server handles cookie clearing now) - setUser(null); - localStorage.removeItem("user"); - sessionStorage.clear(); - - console.log("Logout completed successfully"); + clearLocalSession(); }; - // Small helper components for routing const ProtectedRoute = ({ children }) => { if (!user) return ; return children; }; - const AuthPage = ({ mode }) => { + // Every signed-in page renders inside the same shell (header + sidebar). + const ShellLayout = () => { const navigate = useNavigate(); - const onAuth = (userData) => { - handleAuthenticated(userData); - navigate("/dashboard", { replace: true }); + const onLogoutWrapped = async () => { + await handleLogout(); + navigate("/", { replace: true }); }; - return ; + return ( + + + + ); }; - const DashboardPage = () => { + // The join page must stay reachable without a session: a visitor enters a + // join code first and is sent to login (intent preserved) only when the + // server asks for authentication. + const PublicJoinLayout = () => ( +
+
+
+ + Code4Me research +
+ +
+
+ ); + + const JoinLayout = () => (user ? : ); + + // A deleted account has no session to log out of: the delete response + // cleared the auth cookies, so only the local cleanup runs. + const PrivacyPage = () => { const navigate = useNavigate(); - const onLogoutWrapped = async () => { - await handleLogout(); - navigate("/", { replace: true }); + const onAccountDeleted = () => { + // The router applies navigations as transitions (v7_startTransition). + // Clearing the user in the same transition renders both at once; + // otherwise the signed-out shell would redirect to /login first. + startTransition(() => { + clearLocalSession(); + navigate("/", { replace: true }); + }); + }; + return ; + }; + + const AuthPage = ({ mode }) => { + const navigate = useNavigate(); + const onAuth = (userData) => { + handleAuthenticated(userData); + const pendingJoin = sessionStorage.getItem(RESEARCH_JOIN_INTENT_KEY); + navigate(pendingJoin ? "/research/join" : homePath(userData.user), { replace: true }); }; - return ; + return ; }; - // Show loading screen while checking authentication status if (isLoading) { return ( - - -
-
-
-

Loading...

-
+ +
+
+
+

Loading...

- - +
+ ); } - // Render application with routing return ( - - - -
- - : - } - /> - } /> - } /> - - - - } - /> - } /> - -
- - - + + +
+ + : } + /> + } /> + } /> + + + + } + > + } /> + } /> + } /> + } /> + } /> + } /> + } /> + + }> + } /> + + } /> + +
+ + ); } diff --git a/src/website/src/__tests__/App.routing.test.js b/src/website/src/__tests__/App.routing.test.js new file mode 100644 index 0000000..cc93d8f --- /dev/null +++ b/src/website/src/__tests__/App.routing.test.js @@ -0,0 +1,166 @@ +import React from "react"; +import { fireEvent, render, screen } from "@testing-library/react"; +import App from "../App"; +import * as api from "../utils/api"; + +jest.mock("../utils/api"); +// Isolate the route guards: the pages themselves have their own suites. +jest.mock("../pages/Dashboard", () => () =>
DASHBOARD-PAGE
); +jest.mock("../components/auth/Auth", () => () =>
LOGIN-PAGE
); +jest.mock("../pages/research/ResearchStudies", () => () =>
STUDIES-PAGE
); +jest.mock("../pages/research/ResearchStudyEditor", () => () => ( +
EDITOR-PAGE
+)); +jest.mock("../pages/research/ResearchJoin", () => () =>
JOIN-PAGE
); +jest.mock("../pages/Start", () => () =>
START-PAGE
); +jest.mock("../pages/PrivacySettings", () => ({ onAccountDeleted }) => ( + +)); + +const STUDY_ID = "11111111-1111-1111-1111-111111111111"; + +const renderAt = (path) => { + window.history.pushState({}, "", path); + return render(); +}; + +beforeEach(() => { + jest.clearAllMocks(); + api.getCurrentUser.mockResolvedValue({ + ok: true, + user: { email: "researcher@example.com", is_admin: false }, + config: {}, + }); + api.logoutUser.mockResolvedValue({ ok: true }); +}); + +test("a non-admin authenticated user can reach the study editor route", async () => { + renderAt(`/research/studies/${STUDY_ID}/editor`); + + expect(await screen.findByText("EDITOR-PAGE")).toBeInTheDocument(); + expect(screen.queryByText("DASHBOARD-PAGE")).not.toBeInTheDocument(); +}); + +test("a non-admin authenticated user can reach the studies route", async () => { + renderAt("/research/studies"); + + expect(await screen.findByText("STUDIES-PAGE")).toBeInTheDocument(); +}); + +test("an unauthenticated visitor is redirected to login", async () => { + api.getCurrentUser.mockResolvedValue({ ok: false }); + renderAt(`/research/studies/${STUDY_ID}/editor`); + + expect(await screen.findByText("LOGIN-PAGE")).toBeInTheDocument(); + expect(screen.queryByText("EDITOR-PAGE")).not.toBeInTheDocument(); +}); + +test("registered route tag count matches the route enumeration (ISSUE-007)", () => { + // Pins the App.js route inventory: every ]|]| { + api.getCurrentUser.mockResolvedValue({ + ok: true, + user: { email: "researcher@example.com", is_admin: false, can_research: true }, + config: {}, + }); + renderAt("/"); + + expect(await screen.findByText("STUDIES-PAGE")).toBeInTheDocument(); + expect(window.location.pathname).toBe("/research/studies"); +}); + +test("a signed-in participant opening / lands on My studies", async () => { + renderAt("/"); + + expect(await screen.findByText("JOIN-PAGE")).toBeInTheDocument(); + expect(window.location.pathname).toBe("/research/my-studies"); +}); + +test("a study deep link renders the studies workspace", async () => { + renderAt(`/research/studies/${STUDY_ID}`); + + expect(await screen.findByText("STUDIES-PAGE")).toBeInTheDocument(); +}); + +test("a signed-in participant reaches My studies inside the application shell", async () => { + renderAt("/research/my-studies"); + + expect(await screen.findByText("JOIN-PAGE")).toBeInTheDocument(); + expect(screen.getByRole("navigation", { name: "Main navigation" })).toBeInTheDocument(); +}); + +test("the join page stays reachable without a session", async () => { + api.getCurrentUser.mockResolvedValue({ ok: false }); + renderAt("/research/join"); + + expect(await screen.findByText("JOIN-PAGE")).toBeInTheDocument(); + expect(screen.queryByRole("navigation", { name: "Main navigation" })).not.toBeInTheDocument(); + expect(screen.queryByText("LOGIN-PAGE")).not.toBeInTheDocument(); +}); + +test("only public profile fields are cached locally (never the auth token)", async () => { + api.getCurrentUser.mockResolvedValue({ + ok: true, + user: { + email: "researcher@example.com", + is_admin: false, + auth_token: "live-session-token", + password: "********", + }, + config: {}, + }); + renderAt("/research/studies"); + + expect(await screen.findByText("STUDIES-PAGE")).toBeInTheDocument(); + const stored = localStorage.getItem("user") || ""; + expect(stored).toContain("researcher@example.com"); + expect(stored).not.toContain("live-session-token"); + expect(stored).not.toContain("password"); +}); + +test("a signed-in user reaches Privacy & data inside the application shell", async () => { + renderAt("/settings/privacy"); + + expect(await screen.findByText("PRIVACY-PAGE")).toBeInTheDocument(); + const nav = screen.getByRole("navigation", { name: "Main navigation" }); + expect(nav).toContainElement(screen.getByRole("link", { name: "Privacy & data" })); +}); + +test("Privacy & data requires a session", async () => { + api.getCurrentUser.mockResolvedValue({ ok: false }); + renderAt("/settings/privacy"); + + expect(await screen.findByText("LOGIN-PAGE")).toBeInTheDocument(); + expect(screen.queryByText("PRIVACY-PAGE")).not.toBeInTheDocument(); +}); + +test("a deleted account is signed out locally, without the logout call, and lands on the start page", async () => { + renderAt("/settings/privacy"); + const deleteButton = await screen.findByRole("button", { name: "PRIVACY-PAGE" }); + expect(localStorage.getItem("user")).toContain("researcher@example.com"); + sessionStorage.setItem("code4me.research.join.intent", "JOIN1234"); + + fireEvent.click(deleteButton); + + expect(await screen.findByText("START-PAGE")).toBeInTheDocument(); + expect(window.location.pathname).toBe("/"); + expect(screen.queryByRole("navigation", { name: "Main navigation" })).not.toBeInTheDocument(); + // The same local cleanup as a logout, but no logout request: the session is already gone. + expect(api.logoutUser).not.toHaveBeenCalled(); + expect(localStorage.getItem("user")).toBeNull(); + expect(sessionStorage.getItem("code4me.research.join.intent")).toBeNull(); +}); diff --git a/src/website/src/components/analytics/AgentAnalytics.css b/src/website/src/components/analytics/AgentAnalytics.css new file mode 100644 index 0000000..309524a --- /dev/null +++ b/src/website/src/components/analytics/AgentAnalytics.css @@ -0,0 +1,83 @@ +.agent-analytics { max-width: 1200px; margin: 0 auto; } +.agent-page-header { display: flex; justify-content: space-between; gap: 20px; align-items: end; margin: 0 0 24px; } +.agent-page-header h2 { margin: 4px 0 8px; color: var(--text-color); font-size: 28px; } +.agent-page-header p { margin: 0; color: var(--text-secondary); } +.agent-page-header p.agent-scope-hint { margin-top: 6px; color: var(--primary-color); font-size: 12px; font-weight: 700; } +.agent-eyebrow { color: var(--primary-color); font-size: 11px; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; } +.agent-window { padding: 8px 12px; border: 1px solid var(--border-color); border-radius: 4px; color: var(--text-secondary); font-size: 12px; white-space: nowrap; } +.agent-controls { display: flex; align-items: end; gap: 14px; flex-wrap: wrap; padding: 14px 16px; margin-bottom: 18px; border: 1px solid var(--border-color); border-radius: 8px; background: var(--card-background); } +.agent-controls label { display: flex; flex-direction: column; gap: 5px; color: var(--text-secondary); font-size: 12px; font-weight: 600; } +.agent-controls select { min-width: 170px; padding: 8px; border: 1px solid var(--border-color); border-radius: 4px; background: var(--card-background); color: var(--text-color); } +.agent-clear { padding: 8px 12px; border: 1px solid var(--border-color); border-radius: 4px; background: var(--secondary-color); color: var(--text-color); cursor: pointer; } +.agent-kpis { display: grid; grid-template-columns: repeat(6, minmax(120px, 1fr)); gap: 10px; margin-bottom: 18px; } +.agent-kpis > div { min-height: 100px; padding: 14px; border: 1px solid var(--border-color); border-radius: 8px; background: var(--card-background); box-shadow: 0 2px 8px var(--shadow-color); } +.agent-kpis span, .agent-kpis small { display: block; color: var(--text-secondary); font-size: 11px; } +.agent-kpis strong { display: block; margin: 8px 0 4px; color: var(--text-color); font-size: 23px; } +.agent-token-panel { margin-bottom: 18px; } +.agent-token-grid { display: grid; grid-template-columns: repeat(6, minmax(120px, 1fr)); gap: 10px; } +.agent-token-grid > div { min-height: 82px; padding: 12px; border: 1px solid var(--border-color); border-radius: 6px; background: var(--secondary-color); } +.agent-token-grid span, .agent-token-grid small { display: block; color: var(--text-secondary); font-size: 11px; } +.agent-token-grid strong { display: block; margin: 8px 0 3px; color: var(--text-color); font-size: 19px; } +.agent-layout { display: grid; grid-template-columns: minmax(0, 1.6fr) minmax(280px, 1fr); gap: 18px; } +.agent-analysis-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 18px; margin-bottom: 18px; } +.agent-panel { min-width: 0; padding: 18px; border: 1px solid var(--border-color); border-radius: 8px; background: var(--card-background); box-shadow: 0 2px 8px var(--shadow-color); } +.agent-wide { grid-column: 1 / -1; } +.agent-panel-heading { display: flex; justify-content: space-between; align-items: end; gap: 12px; margin-bottom: 14px; } +.agent-panel h3 { margin: 3px 0 0; color: var(--text-color); font-size: 17px; } +.agent-muted { color: var(--text-secondary); font-size: 11px; } +.agent-table-scroll { overflow-x: auto; } +.agent-runs-scroll { max-height: 360px; overflow-y: auto; } +.agent-table { width: 100%; border-collapse: collapse; color: var(--text-color); font-size: 12px; } +.agent-table th { padding: 10px 8px; background: var(--secondary-color); color: var(--text-secondary); text-align: left; font-size: 10px; text-transform: uppercase; letter-spacing: .04em; white-space: nowrap; } +.agent-table td { padding: 11px 8px; border-bottom: 1px solid var(--border-color); white-space: nowrap; } +.agent-table td small { display: block; margin-top: 3px; color: var(--text-secondary); font-size: 10px; } +.agent-rate { font-weight: 700; } +.agent-status { display: inline-block; padding: 3px 7px; border-radius: 4px; font-size: 10px; font-weight: 700; text-transform: uppercase; } +.agent-status.done { background: rgba(21, 128, 61, .14); color: #15803d; } +.agent-status.failed { background: rgba(194, 65, 12, .14); color: #c2410c; } +.agent-status.pending, .agent-status.running { background: rgba(202, 138, 4, .14); color: #a16207; } +.agent-tool-list { display: grid; gap: 8px; } +.agent-tool { padding: 10px 0; border-bottom: 1px solid var(--border-color); } +.agent-tool > div { display: flex; justify-content: space-between; gap: 10px; } +.agent-tool > div + div { margin-top: 5px; color: var(--text-secondary); font-size: 11px; } +.agent-bad { color: #c2410c; } +.agent-good { color: #15803d; } +.agent-empty { margin: 0; color: var(--text-secondary); font-size: 13px; } +.agent-panel .chart-container { height: 210px; margin: 0; } +.agent-distribution { display: grid; gap: 13px; padding: 8px 0; } +.agent-distribution-row { display: grid; grid-template-columns: 76px 1fr 34px; gap: 8px; align-items: center; color: var(--text-secondary); font-size: 11px; } +.agent-distribution-row > div { height: 10px; overflow: hidden; border-radius: 3px; background: var(--secondary-color); } +.agent-distribution-row i { display: block; height: 100%; border-radius: inherit; background: #0f766e; } +.agent-distribution-row strong { color: var(--text-color); text-align: right; } +.agent-signal-list { display: grid; gap: 8px; } +.agent-signal-list div { display: flex; justify-content: space-between; gap: 12px; padding: 8px 0; border-bottom: 1px solid var(--border-color); color: var(--text-secondary); font-size: 12px; } +.agent-signal-list strong { color: var(--text-color); } +.agent-error-list strong { color: #c2410c; } +.agent-inspect, .agent-close { padding: 5px 9px; border: 1px solid var(--border-color); border-radius: 4px; background: var(--secondary-color); color: var(--text-color); cursor: pointer; font-size: 11px; } +.agent-inspector { margin-top: 18px; padding: 18px; border: 1px solid var(--primary-color); border-radius: 8px; background: var(--card-background); box-shadow: 0 4px 16px var(--shadow-color); } +.agent-inspector-heading { display: flex; justify-content: space-between; align-items: end; gap: 12px; margin-bottom: 14px; } +.agent-inspector-heading .agent-panel-heading { margin-bottom: 0; } +.agent-event-scroll { max-height: 420px; overflow-y: auto; padding: 4px 6px 4px 0; } +.agent-event-graph { display: grid; gap: 6px; min-width: 420px; } +.agent-graph-row { position: relative; display: flex; gap: 8px; min-height: 48px; } +.agent-graph-rail { position: relative; width: 12px; flex: 0 0 12px; } +.agent-graph-rail::before { position: absolute; top: 0; bottom: -6px; left: 5px; width: 2px; background: var(--border-color); content: ""; } +.agent-graph-rail::after { position: absolute; top: 18px; left: 1px; width: 10px; height: 10px; border: 2px solid var(--card-background); border-radius: 50%; background: #0f766e; box-shadow: 0 0 0 1px #0f766e; content: ""; } +.agent-graph-rail.tool::after { background: #d97706; box-shadow: 0 0 0 1px #d97706; } +.agent-graph-rail.observed::after { background: #6d28d9; box-shadow: 0 0 0 1px #6d28d9; } +.agent-graph-rail.failure::after { background: #c2410c; box-shadow: 0 0 0 1px #c2410c; } +.agent-graph-node { display: flex; justify-content: space-between; gap: 14px; width: 100%; padding: 8px 10px; border: 1px solid rgba(15, 118, 110, .25); border-radius: 6px; background: rgba(15, 118, 110, .06); } +.agent-graph-node.tool { border-color: rgba(217, 119, 6, .3); background: rgba(217, 119, 6, .06); } +.agent-graph-node.observed { border-color: rgba(109, 40, 217, .3); background: rgba(109, 40, 217, .06); } +.agent-graph-node.failure { border-color: rgba(194, 65, 12, .4); background: rgba(194, 65, 12, .07); } +.agent-graph-node-main { min-width: 0; } +.agent-graph-node-main strong, .agent-graph-node-main small { display: block; } +.agent-graph-node-main strong { color: var(--text-color); font-size: 12px; } +.agent-graph-node-main small { margin-top: 3px; overflow: hidden; color: var(--text-secondary); font-size: 10px; text-overflow: ellipsis; white-space: nowrap; } +.agent-graph-node-stats { display: flex; flex: 0 0 auto; align-items: center; gap: 8px; color: var(--text-secondary); font-size: 10px; white-space: nowrap; } +.agent-run-meta { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 14px; color: var(--text-secondary); font-size: 12px; } +.agent-run-meta > span:not(.agent-status) { padding: 5px 8px; border-radius: 4px; background: var(--secondary-color); } +.agent-event-index { color: var(--primary-color); font-variant-numeric: tabular-nums; } +.agent-error-text { color: #c2410c; } +@media (max-width: 1050px) { .agent-kpis, .agent-token-grid { grid-template-columns: repeat(3, minmax(140px, 1fr)); } } +@media (max-width: 680px) { .agent-page-header, .agent-panel-heading { align-items: start; flex-direction: column; } .agent-kpis, .agent-layout, .agent-analysis-grid { grid-template-columns: 1fr; } .agent-wide { grid-column: auto; } .agent-controls select { width: 100%; } .agent-event-graph { min-width: 0; } .agent-graph-node { align-items: start; flex-direction: column; gap: 5px; } .agent-graph-node-stats { flex-wrap: wrap; } } diff --git a/src/website/src/components/analytics/AgentAnalytics.js b/src/website/src/components/analytics/AgentAnalytics.js new file mode 100644 index 0000000..1c358f9 --- /dev/null +++ b/src/website/src/components/analytics/AgentAnalytics.js @@ -0,0 +1,237 @@ +import React, { useEffect, useState } from "react"; +import Chart from "../visualization/Chart"; +import { getAgentOverview, getAgentRunDetail } from "../../utils/api"; +import "./AgentAnalytics.css"; + +const number = (value) => Number(value || 0).toLocaleString(); +const percent = (value) => `${(Number(value || 0) * 100).toFixed(1)}%`; +const milliseconds = (value) => `${Math.round(Number(value || 0))}ms`; +// A missing measurement is rendered as unavailable, never as a zero. +const ifAvailable = (value, render) => (value == null ? "unavailable" : render(value)); + +const mergeOptions = (previous, values) => [...new Set([...previous, ...values])].sort(); + +const FilterSelect = ({ label, value, options, onChange, allLabel }) => ( + +); + +const PanelHeading = ({ eyebrow, title, hint }) => ( +
+
{eyebrow}

{title}

+ {hint && {hint}} +
+); + +const KpiCard = ({ label, value, detail }) => ( +
{label}{value}{detail}
+); + +const eventDepth = (event, spanIndexes, events) => { + if (!event.parent_span_id) return 0; + let parentIndex = spanIndexes.get(event.parent_span_id); + let depth = 0; + while (parentIndex !== undefined && depth < 4) { + depth += 1; + const parent = events[parentIndex]; + parentIndex = spanIndexes.get(parent?.parent_span_id); + } + return depth; +}; + +const EventGraph = ({ events }) => { + const spanIndexes = new Map(events.map((event, index) => [event.span_id, index])); + return ( +
+
+ {events.map((event) => { + const depth = eventDepth(event, spanIndexes, events); + const isTool = event.event_type.includes("tool"); + const isFailure = event.event_type.includes("failed") || event.upstream_status >= 400; + const isObserved = event.source === "acp-proxy"; + const kind = isObserved ? "observed" : isTool ? "tool" : "model"; + return ( +
+
+
+
+ {event.event_index} + {event.event_type} + {event.source || "unknown source"}{event.detail ? ` · ${event.detail}` : ""}{event.tool_name ? ` · ${event.tool_name}` : event.model ? ` · ${event.model}` : ""} +
+
+ {event.latency_ms ? {milliseconds(event.latency_ms)} : null} + {event.total_tokens ? {number(event.total_tokens)} tokens : null} + {event.upstream_status ? HTTP {event.upstream_status} : null} +
+
+
+ ); + })} +
+
+ ); +}; + +const AgentAnalytics = ({ timeWindow = "7d" }) => { + const [data, setData] = useState(null); + const [framework, setFramework] = useState(""); + const [model, setModel] = useState(""); + const [profile, setProfile] = useState(""); + const [filterOptions, setFilterOptions] = useState({ + frameworks: [], + models: [], + profiles: [], + }); + const [selectedRun, setSelectedRun] = useState(null); + const [runDetail, setRunDetail] = useState(null); + const [isRunLoading, setIsRunLoading] = useState(false); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + const load = async () => { + setIsLoading(true); + setError(null); + const params = { time_window: timeWindow }; + if (framework) params.framework = framework; + if (model) params.model = model; + if (profile) params.profile = profile; + const response = await getAgentOverview(params); + if (cancelled) return; + if (response.ok) { + setData(response.data); + const nextProfiles = response.data.profiles || []; + setFilterOptions((previous) => ({ + frameworks: mergeOptions(previous.frameworks, nextProfiles.map((item) => item.framework_version)), + models: mergeOptions(previous.models, nextProfiles.map((item) => item.model)), + profiles: mergeOptions(previous.profiles, nextProfiles.map((item) => item.profile_name)), + })); + } else setError(response.error); + setIsLoading(false); + }; + load().catch((loadError) => { + console.error("Agent telemetry error:", loadError); + setError("Failed to load agent telemetry"); + setIsLoading(false); + }); + return () => { cancelled = true; }; + }, [timeWindow, framework, model, profile]); + + const openRun = async (taskId) => { + setSelectedRun(taskId); + setIsRunLoading(true); + const response = await getAgentRunDetail(taskId); + setRunDetail(response.ok ? response.data : { error: response.error }); + setIsRunLoading(false); + }; + + if (isLoading) { + return

Loading Agent Telemetry...

; + } + + if (error) { + return

Failed to load agent telemetry

{error}

; + } + + const summary = data?.summary || {}; + const profiles = data?.profiles || []; + const tools = data?.tools || []; + const runs = data?.recent_runs || []; + const trend = data?.trend || []; + const latencyDistribution = data?.latency_distribution || []; + const maxLatencyCalls = Math.max(...latencyDistribution.map((item) => item.calls), 1); + const eventTypes = data?.event_types || []; + const errors = data?.errors || []; + const frameworks = filterOptions.frameworks; + const models = filterOptions.models; + const availableProfiles = filterOptions.profiles; + + return ( +
+
+
+ Operational telemetry +

Agent Performance

+

Understand how agents run, where they spend time, and where execution fails.

+

Your own agent runs — personal scope

+
+
Last {timeWindow.replace("d", " days")}
+
+ +
+ + + + {(framework || model || profile) && } +
+ +
+ + + + + + +
+ +
+ +
+
Provider input tokens{ifAvailable(summary.provider_input_tokens, number)}reported by model provider
+
Conversation context{ifAvailable(summary.conversation_tokens_estimated, (v) => `~${number(v)}`)}estimated from message bytes
+
Tool schema{ifAvailable(summary.tool_schema_tokens_estimated, (v) => `~${number(v)}`)}estimated from schema bytes
+
Tool results{ifAvailable(summary.tool_result_tokens_estimated, (v) => `~${number(v)}`)}estimated from result bytes
+
Model output{ifAvailable(summary.model_output_tokens, number)}reported completion tokens
+
Successful calls{number(summary.successful_model_calls)}{number(summary.rate_limit_retries)} rate-limit retries
+
+
+ +
+
+ + {trend.length ? ({ label: item.day.slice(5, 10), value: item.runs }))} title="Runs per day" color="#0f766e" xKey="label" xType="category" /> :

No trend data in this time range.

} +
+
+ + {latencyDistribution.length ?
{latencyDistribution.map((item) =>
{item.bucket}
{number(item.calls)}
)}
:

No latency data in this time range.

} +
+
+ + {eventTypes.length ?
{eventTypes.slice(0, 8).map((item) =>
{item.event_type}{number(item.events)}
)}
:

No event data in this time range.

} +
+
+ + {errors.length ?
{errors.map((item) =>
{item.reason}{number(item.events)}
)}
:

No recorded failures in this time range.

} +
+
+ +
+
+ + {profiles.length ?
{profiles.map((item) => )}
ProfileRuntimeModelRunsCompletionFailedAvg steps
{item.profile_name}{item.framework_version}{item.model}{number(item.tasks)}{percent(item.completion_rate)}{number(item.failed)}{Number(item.avg_steps || 0).toFixed(1)}
:

No profile telemetry in this time range.

} +
+ +
+ + {tools.length ?
{tools.map((item) =>
{item.tool_name}{number(item.calls)} calls
{milliseconds(item.avg_latency_ms)}{number(item.failures)} failed
)}
:

No tool telemetry in this time range.

} +
+ +
+ + {runs.length ?
{runs.map((run) => )}
StatusProfileModelCreatedStepsModel callsTool callsFailures
{run.status}{run.profile_name || "unknown"}{run.framework_version}{run.model}{run.created_at ? new Date(run.created_at).toLocaleString() : "n/a"}{number(run.steps)}{number(run.model_calls)}{number(run.tool_calls)}{number(run.failures)}
:

No agent runs in this time range.

} +
+
+ + {selectedRun &&
{isRunLoading &&

Loading run events...

}{runDetail?.error &&

{runDetail.error}

}{runDetail?.run &&
{runDetail.run.profile_name}{runDetail.run.model}{runDetail.run.status}
}{runDetail?.events?.length > 0 && }{runDetail?.events?.length === 0 && !isRunLoading &&

This run has no persisted events.

}
} +
+ ); +}; + +export default AgentAnalytics; diff --git a/src/website/src/components/analytics/AnalyticsNavigation.css b/src/website/src/components/analytics/AnalyticsNavigation.css deleted file mode 100644 index 67455ac..0000000 --- a/src/website/src/components/analytics/AnalyticsNavigation.css +++ /dev/null @@ -1,124 +0,0 @@ -.analytics-navigation { - background-color: var(--card-background); - border-radius: 8px; - padding: 20px; - margin-bottom: 20px; - box-shadow: 0 2px 8px var(--shadow-color); -} - -.nav-header { - display: flex; - justify-content: space-between; - align-items: center; - margin-bottom: 20px; - padding-bottom: 15px; - border-bottom: 1px solid var(--border-color); -} - -.nav-header h2 { - margin: 0; - color: var(--text-color); - font-size: 18px; - font-weight: 600; -} - -.admin-badge { - background: linear-gradient(135deg, #ff9800, #f57c00); - color: white; - padding: 4px 8px; - border-radius: 12px; - font-size: 11px; - font-weight: 600; - text-transform: uppercase; - letter-spacing: 0.5px; - box-shadow: 0 2px 4px rgba(255, 152, 0, 0.3); -} - -.nav-items { - display: flex; - gap: 8px; - flex-wrap: wrap; -} - -.nav-item { - display: flex; - align-items: center; - gap: 12px; - padding: 12px 16px; - background-color: var(--secondary-color); - border: 1px solid var(--border-color); - border-radius: 6px; - cursor: pointer; - transition: all 0.2s ease; - flex: 1; - min-width: 200px; -} - -.nav-item:hover { - background-color: var(--hover-color); - border-color: var(--primary-color); - transform: translateY(-1px); - box-shadow: 0 2px 4px var(--shadow-color); -} - -.nav-item.active { - background-color: var(--primary-color); - color: white; - border-color: var(--primary-color); - box-shadow: 0 2px 8px rgba(66, 133, 244, 0.3); -} - -.nav-item.active .nav-label, -.nav-item.active .nav-description { - color: white; -} - -.nav-item.admin-only { - border-left: 3px solid #ff9800; -} - -.nav-icon { - font-size: 20px; - flex-shrink: 0; -} - -.nav-content { - display: flex; - flex-direction: column; - gap: 2px; - flex: 1; -} - -.nav-label { - font-weight: 500; - color: var(--text-color); - font-size: 14px; -} - -.nav-description { - font-size: 11px; - color: var(--text-secondary); - line-height: 1.3; -} - -.admin-indicator { - font-size: 14px; - opacity: 0.8; -} - -/* Responsive design */ -@media (max-width: 768px) { - .nav-items { - flex-direction: column; - } - - .nav-item { - min-width: unset; - } - - .nav-header { - flex-direction: column; - gap: 10px; - align-items: flex-start; - } -} diff --git a/src/website/src/components/analytics/AnalyticsNavigation.js b/src/website/src/components/analytics/AnalyticsNavigation.js deleted file mode 100644 index 44acfc0..0000000 --- a/src/website/src/components/analytics/AnalyticsNavigation.js +++ /dev/null @@ -1,82 +0,0 @@ -import React from 'react'; -import './AnalyticsNavigation.css'; - -const AnalyticsNavigation = ({ activeView, onViewChange, user }) => { - const navigationItems = [ - { - id: 'overview', - label: 'Dashboard Overview', - icon: '📊', - description: 'Key metrics and trends' - }, - { - id: 'usage', - label: 'Usage Analytics', - icon: '📈', - description: 'Query patterns and user behavior' - }, - { - id: 'models', - label: 'Model Performance', - icon: '🤖', - description: 'AI model comparison and quality metrics' - }, - { - id: 'calibration', - label: 'Model Calibration', - icon: '🎯', - description: 'Confidence and reliability analysis' - } - ]; - - // Add admin-only navigation items - if (user?.is_admin) { - navigationItems.push({ - id: 'studies', - label: 'A/B Testing', - icon: '🔬', - description: 'Experiment management and evaluation', - adminOnly: true - }); - navigationItems.push({ - id: 'configs', - label: 'Config Management', - icon: '🧩', - description: 'Create and manage server/config modules', - adminOnly: true - }); - } - - return ( - - ); -}; - -export default AnalyticsNavigation; diff --git a/src/website/src/components/analytics/CalibrationAnalytics.css b/src/website/src/components/analytics/CalibrationAnalytics.css index 4c351c5..403156a 100644 --- a/src/website/src/components/analytics/CalibrationAnalytics.css +++ b/src/website/src/components/analytics/CalibrationAnalytics.css @@ -144,3 +144,21 @@ @keyframes spin { to { transform: rotate(360deg); } } + +/* Per-model calibration summary table. */ +.summary-section { + background-color: var(--card-background); + border: 1px solid var(--border-color); + border-radius: 12px; + display: flex; + flex-direction: column; + gap: 12px; + margin-top: 20px; + padding: 20px; +} + +.summary-section h3 { + color: var(--text-color); + font-size: 16px; + margin: 0; +} diff --git a/src/website/src/components/analytics/ConfigManagement.css b/src/website/src/components/analytics/ConfigManagement.css index ad5c789..76c5ecd 100644 --- a/src/website/src/components/analytics/ConfigManagement.css +++ b/src/website/src/components/analytics/ConfigManagement.css @@ -364,4 +364,11 @@ overflow: auto; max-height: 420px; line-height: 1.35; -} \ No newline at end of file +} +/* Editor header action row (mode toggle, errors, save). */ +.editor-header .actions { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: 10px; +} diff --git a/src/website/src/components/analytics/ConfigManagement.js b/src/website/src/components/analytics/ConfigManagement.js index c568950..59e658c 100644 --- a/src/website/src/components/analytics/ConfigManagement.js +++ b/src/website/src/components/analytics/ConfigManagement.js @@ -1,4 +1,4 @@ -import React, { useEffect, useMemo, useState } from 'react'; +import React, { useEffect, useState } from 'react'; import { listConfigs, getConfigById, @@ -7,7 +7,6 @@ import { deleteConfigApi, getLanguagesMapping, getAvailableModels, - validateHuggingFaceModel, } from '../../utils/api'; import HOCON from 'hocon-parser'; import './ConfigManagement.css'; @@ -27,7 +26,6 @@ const emptyModulesTemplate = () => ({ }); const defaultServer = { host: 'http://127.0.0.1', port: 8008, contextPath: '', timeout: 5000 }; -const defaultAuth = { google: { clientId: '' } }; const defaultModels = { available: [], systemPrompt: '' }; // Simple HOCON serializer (object -> HOCON string) @@ -88,7 +86,6 @@ const STARTER_HOCON = `config { } } server { host = "http://127.0.0.1" port = 8008 contextPath = "" timeout = 5000 } - auth { google { clientId = "" } } models { available = [] systemPrompt = "" } // languages: static, provided by the system (you can omit this block) }`; @@ -155,8 +152,6 @@ const SCHEMA_GUIDE = `config { // Server Settings server { host = "http://127.0.0.1" port = 8008 contextPath = "" timeout = 5000 } - // Authentication Settings - auth { google { clientId = "" } } // Model configuration models { @@ -171,28 +166,7 @@ const SCHEMA_GUIDE = `config { // languages { ... } }`; -const getConfigRoot = (parsed) => { - if (parsed && typeof parsed === 'object') { - if (parsed.config && typeof parsed.config === 'object') return parsed.config; - } - return parsed || {}; -}; -const normalizeParsedToForm = (parsedRoot, languagesMapping) => { - const safe = (v, d) => (v === undefined || v === null ? d : v); - const server = safe(parsedRoot.server, defaultServer); - const auth = safe(parsedRoot.auth, defaultAuth); - const models = safe(parsedRoot.models, defaultModels); - const modules = safe(parsedRoot.modules, emptyModulesTemplate().modules); - return { - modules, - server, - auth, - models, - // Always enforce static languages from backend - languages: languagesMapping || {}, - }; -}; const parseHoconSafe = (text) => { try { @@ -207,136 +181,23 @@ const parseHoconSafe = (text) => { throw new Error('Failed to parse HOCON'); }; -function ModuleCard({ mod, onChange, onRemove }) { - const [expanded, setExpanded] = useState(true); - - const updateField = (field, value) => onChange({ ...mod, [field]: value }); - const updateSubmodule = (idx, value) => { - const submodules = [...(mod.submodules || [])]; - submodules[idx] = value; onChange({ ...mod, submodules }); - }; - const addSubmodule = () => onChange({ ...mod, submodules: [...(mod.submodules || []), { id: '', class: '', name: '', type: '', description: '', enabled: true }] }); - const removeSubmodule = (idx) => { - const submodules = (mod.submodules || []).filter((_, i) => i !== idx); - onChange({ ...mod, submodules }); - }; - - const updateDependency = (idx, value) => { - const dependencies = [...(mod.dependencies || [])]; - dependencies[idx] = value; onChange({ ...mod, dependencies }); - }; - const addDependency = () => onChange({ ...mod, dependencies: [...(mod.dependencies || []), { moduleId: '', isHard: false }] }); - const removeDependency = (idx) => onChange({ ...mod, dependencies: (mod.dependencies || []).filter((_, i) => i !== idx) }); - - return ( -
-
setExpanded(!expanded)}> -
{mod.name || mod.id || 'New Module'}
-
- - -
-
- {expanded && ( -
-
- - - - -
-