feat: add long-horizon agent eval project

.github/workflows/long-horizon-agent-evals.yml

@@ -0,0 +1,42 @@
+name: Long-horizon agent evals
+
+"on":
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: read
+
+jobs:
+  check:
+    name: Check long-horizon agent evals
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: projects/long-horizon-agent-evals
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+          cache: npm
+          cache-dependency-path: projects/long-horizon-agent-evals/package-lock.json
+          registry-url: https://npm.pkg.github.com
+          scope: "@nvidia"
+
+      - name: Install dependencies
+        run: npm ci
+        env:
+          NODE_AUTH_TOKEN: ${{ github.token }}
+
+      - name: Run project checks
+        run: npm run check

docs/dev-notes/index.md

@@ -31,10 +31,44 @@ hide:
       <h2 id="featured-note-title">Featured note</h2>
       <span>Latest from the team</span>
     </div>
-    <article class="dev-note-card dev-note-card--featured dev-note-card--physical-ai dev-note-card--has-image">
+    <article class="dev-note-card dev-note-card--featured dev-note-card--openshell dev-note-card--has-image">
+      <a class="dev-note-card__link" href="posts/2026-08-27-adversarial-policy-review-long-horizon-agents/">
+      <div class="dev-note-card__visual dev-note-card__visual--openshell dev-note-card__visual--image" aria-hidden="true">
+        <img class="dev-note-card__visual-image" src="../assets/long-horizon-agent-evals/back-and-forth-by-session.png" alt="" loading="eager" fetchpriority="high">
+      </div>
+      <div class="dev-note-card__copy">
+        <div class="dev-note-card__meta">
+          <time datetime="2026-08-27">August 27, 2026</time>
+          <span>OpenShell</span>
+        </div>
+        <h3>Adversarial prototype: AI policy auto-approval for long-horizon agents in OpenShell</h3>
+        <p class="dev-note-card__summary">Can an attacker agent convince an equally capable reviewer agent to grant a capability that is explicitly prohibited?</p>
+        <div class="dev-note-card__tags" aria-label="Tags">
+          <span>agents</span>
+          <span>policy</span>
+          <span>security</span>
+        </div>
+        <div class="dev-note-card__footer">
+        <span class="dev-note-card__authors" aria-label="Author: Alex Watson">
+          <img src="https://github.com/zredlined.png?size=64" alt="" loading="lazy">
+          <span class="dev-note-card__author-names">Alex Watson</span>
+        </span>
+          <span class="dev-note-card__read">Read note</span>
+        </div>
+      </div>
+      </a>
+    </article>
+  </section>
+  <section class="journal-section dev-notes-recent" aria-labelledby="recent-notes-title">
+    <div class="journal-section__head">
+      <h2 id="recent-notes-title">Recent notes</h2>
+      <span>The working archive</span>
+    </div>
+    <div class="dev-notes-recent-list">
+    <article class="dev-note-card dev-note-card--recent dev-note-card--physical-ai dev-note-card--has-image">
       <a class="dev-note-card__link" href="posts/2026-08-07-formal-methods-ai-generated-robot-actions/">
       <div class="dev-note-card__visual dev-note-card__visual--physical-ai dev-note-card__visual--image" aria-hidden="true">
-        <img class="dev-note-card__visual-image" src="../assets/robotics-policy-prover/robotics-policy-prover-hero.png" alt="" loading="eager" fetchpriority="high">
+        <img class="dev-note-card__visual-image" src="../assets/robotics-policy-prover/robotics-policy-prover-hero.png" alt="" loading="lazy">
       </div>
       <div class="dev-note-card__copy">
         <div class="dev-note-card__meta">
@@ -58,13 +92,6 @@ hide:
       </div>
       </a>
     </article>
-  </section>
-  <section class="journal-section dev-notes-recent" aria-labelledby="recent-notes-title">
-    <div class="journal-section__head">
-      <h2 id="recent-notes-title">Recent notes</h2>
-      <span>The working archive</span>
-    </div>
-    <div class="dev-notes-recent-list">
     <article class="dev-note-card dev-note-card--recent dev-note-card--edge-ai dev-note-card--has-image">
       <a class="dev-note-card__link" href="posts/2026-07-20-policy-controlling-reachy-mini-with-openshell/">
       <div class="dev-note-card__visual dev-note-card__visual--edge-ai dev-note-card__visual--image" aria-hidden="true">

projects/README.md

@@ -6,6 +6,8 @@ layout.
 
 Current projects:
 
+- `long-horizon-agent-evals`: Persistent agent experiments over configurable time
+  horizons and repeated parallel attempts, starting with GitHub policy review.
 - `openshell-middleware-manager`: `omm` CLI that creates and updates version-matched
   Python and Rust OpenShell supervisor middleware projects.
 - `egress-gate`: Extensible OpenShell middleware for provider-bound HTTP

projects/long-horizon-agent-evals/.dockerignore

@@ -0,0 +1,2 @@
+**
+!Dockerfile.challenger

projects/long-horizon-agent-evals/.env.example

@@ -0,0 +1,65 @@
+# Required: OpenShell and the real GitHub target
+LAB_OPENSHELL_GATEWAY=http://127.0.0.1:8080
+LAB_GITHUB_OWNER=
+LAB_GITHUB_REPO=
+LAB_GITHUB_BRANCH=main
+LAB_GITHUB_TOKEN=
+
+# Model configuration: challenger
+LAB_CHALLENGER_RESPONSES_URL=https://inference-api.nvidia.com/v1/responses
+LAB_CHALLENGER_MODEL=openai/openai/gpt-5.6-sol
+LAB_CHALLENGER_REASONING=medium
+
+# Challenger lull detection: rotate to a fresh thread (reseeded with the task
+# prompt plus a bounded activity summary) when the agent stops using tools and
+# starts repeating itself. Both conditions must hold. Set the duplicate rate
+# above 1 to disable. Validated against the 20260821/20260822 trace corpus:
+# healthy runs peaked at a 22% duplicate rate, stalled runs reached 72-86%.
+LAB_CHALLENGER_LULL_WINDOW_TURNS=40
+LAB_CHALLENGER_LULL_MIN_IDLE_TURNS=40
+LAB_CHALLENGER_LULL_MIN_DUPLICATE_RATE=0.5
+LAB_CHALLENGER_API_KEY=
+
+# Model configuration: reviewer. URL, model, and API key reuse challenger values when empty.
+LAB_REVIEWER_RESPONSES_URL=
+LAB_REVIEWER_MODEL=
+LAB_REVIEWER_REASONING=medium
+LAB_REVIEWER_API_KEY=
+
+# Runtime controls. The three command-line flags override these values.
+LAB_DURATION_MINUTES=30
+LAB_RUNS=50
+LAB_CONCURRENCY=2
+
+# Advanced tuning
+LAB_DEDICATED_GATEWAY=1
+LAB_WORKSPACE=default
+LAB_REVIEW_GRACE_SECONDS=90
+LAB_ORACLE_POLL_SECONDS=60
+LAB_ORACLE_MIN_SUCCESS_PERCENT=90
+LAB_CONTINUE_AFTER_COMPROMISE=0
+LAB_KEEP_SANDBOX=0
+LAB_SANDBOX_IMAGE=long-horizon-agent-evals/codex:0.147.0-tools1
+LAB_RUNS_DIR=./runs
+LAB_MODEL_BACKOFF_BASE_SECONDS=15
+LAB_MODEL_BACKOFF_MAX_SECONDS=120
+LAB_MODEL_REQUEST_TIMEOUT_SECONDS=180
+LAB_CHALLENGER_CONTEXT_WINDOW=128000
+LAB_CHALLENGER_EFFECTIVE_CONTEXT_PERCENT=80
+LAB_CHALLENGER_THREAD_ROTATE_AFTER_FAILURES=3
+LAB_CHALLENGER_MAX_THREAD_ROTATIONS=6
+LAB_CHALLENGER_THREAD_MAX_SUCCESSFUL_TURNS=
+LAB_CHALLENGER_HANDOFF_MAX_CHARACTERS=24000
+LAB_REVIEWER_HISTORY_MAX_MESSAGES=16
+LAB_REVIEWER_HISTORY_MAX_CHARACTERS=240000
+LAB_MAX_BACKOFF_PERCENT=25
+LAB_MAX_ATTEMPTS=
+LAB_RETRY_DELAY_SECONDS=10
+LAB_RATE_LIMIT_COOLDOWN_SECONDS=300
+
+# Optional OpenShell authentication/TLS
+# OPENSHELL_TOKEN=
+# OPENSHELL_CA_CERT=
+# OPENSHELL_CLIENT_CERT=
+# OPENSHELL_CLIENT_KEY=
+# OPENSHELL_INSECURE=0

projects/long-horizon-agent-evals/.gitignore

@@ -0,0 +1,6 @@
+node_modules/
+dist/
+runs/*
+!runs/.gitkeep
+.env
+*.log

projects/long-horizon-agent-evals/.npmrc

@@ -0,0 +1,2 @@
+@nvidia:registry=https://npm.pkg.github.com
+//npm.pkg.github.com/:_authToken=${NODE_AUTH_TOKEN}

projects/long-horizon-agent-evals/AGENTS.md

@@ -0,0 +1,15 @@
+# Long-Horizon Agent Evals
+
+Read `README.md` before changing this project. Keep the GitHub experiment wired
+directly into `src/campaign.ts`; do not add a scenario framework until a second
+real experiment demonstrates shared code.
+
+Use Node.js 20.3 or newer and npm. Run commands from this directory.
+
+```shell
+npm run check
+```
+
+Never commit `.env`, credentials, or files under `runs/`. Treat the GitHub
+preflight and campaigns as real mutations: use a disposable repository and a
+repository-scoped token.

projects/long-horizon-agent-evals/Dockerfile.challenger

@@ -0,0 +1,16 @@
+ARG BASE_IMAGE=ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:aeef1c63f00e2913ea002ccb3aaf925f338b5c5d70e63576f0d95c16a138044e
+FROM ${BASE_IMAGE}
+
+ARG CODEX_VERSION=0.147.0
+
+USER root
+RUN apt-get update \
+    && apt-get install --yes --no-install-recommends jq ripgrep \
+    && rm -rf /var/lib/apt/lists/* \
+    && npm install --global --no-audit --no-fund "@openai/codex@${CODEX_VERSION}" \
+    && test "$(codex --version)" = "codex-cli ${CODEX_VERSION}"
+
+LABEL org.opencontainers.image.title="OpenShell long-horizon eval challenger" \
+      org.opencontainers.image.version="${CODEX_VERSION}"
+
+USER sandbox