From f1b913991ca65e53f5e93d9bbfa3179ec2dcf9f5 Mon Sep 17 00:00:00 2001 From: Pilipilis Date: Tue, 6 Oct 2026 15:26:38 +0000 Subject: [PATCH 1/5] fix: make webhook intent fallback conservative Co-authored-by: Eduard Carreras --- docs/operations.md | 9 +- docs/policy-reference.md | 13 +-- src/github_agent_bridge/intent_classifier.py | 72 +++++++++------- src/github_agent_bridge/queue.py | 12 ++- tests/test_intent_classifier.py | 61 ++++++++++++-- tests/test_queue.py | 88 ++++++++++++++++++++ tests/test_webhook.py | 2 +- 7 files changed, 208 insertions(+), 49 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index 941ab68c..784075b0 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -79,10 +79,11 @@ count. Otherwise four busy workers can fill a four-slot OpenClaw queue and starve other gateway work. The enqueue-time intent classifier is deliberately different: the bridge calls -`openclaw agent --local` for an isolated one-shot classification. This keeps the -reader independent from gateway queue saturation, event-loop stalls, and -gateway SQLite lock contention. Classifier calls are sequential in the reader, -so they do not add another pool of concurrent bridge jobs. +`openclaw agent exec` for a one-shot classification. This avoids depending on +the long-lived Gateway session path for the permission decision. The bridge +limits classifier subprocess concurrency, retries once, and treats webhook +comment/review classifier failure as `review_only`, so a classifier outage cannot +elevate webhook work to `work_allowed`. For the standard four-worker deployment, use eight OpenClaw slots: diff --git a/docs/policy-reference.md b/docs/policy-reference.md index 8cc08284..7b72205c 100644 --- a/docs/policy-reference.md +++ b/docs/policy-reference.md @@ -759,12 +759,13 @@ Agents must also apply the comment value rule before posting: comment only when ## Intent classifier -`intentClassifier` controls an optional enqueue-time LLM classifier for trusted GitHub comments and reviews. It is disabled by default; when enabled, the bridge calls OpenClaw with the packaged `prompt_rules/intent_classifier.md` prompt or `promptOverrides.rules.intent_classifier`, expects JSON output, and uses the result only when confidence is high enough. Low-confidence, invalid, timed-out, or failed classifier calls fall back to the deterministic parser result. +`intentClassifier` controls an optional enqueue-time LLM classifier for trusted GitHub comments and reviews. It is disabled by default; when enabled, the bridge calls OpenClaw with the packaged `prompt_rules/intent_classifier.md` prompt or `promptOverrides.rules.intent_classifier`, expects JSON output, and uses the result only when confidence is high enough. Low-confidence, invalid, timed-out, or failed webhook comment/review classifier calls fall back to `review_only`; email ingestion keeps the deterministic parser result for backward compatibility. -Classifier calls use `openclaw agent --local`, isolating enqueue-time routing -from gateway concurrency, event-loop stalls, and gateway SQLite locks. Normal -executor, feedback-learning, and interactive gateway calls still need suitable -OpenClaw concurrency headroom; see +Classifier calls use `openclaw agent exec`, avoiding the long-lived Gateway +session path for enqueue-time routing. The bridge retries once, limits classifier +subprocess concurrency, and treats failure as `review_only` for webhook +comments/reviews. Normal executor, feedback-learning, and interactive gateway +calls still need suitable OpenClaw concurrency headroom; see [`operations.md`](operations.md#openclaw-concurrency-headroom). The classifier returns structured semantics: whether the event is addressed to the configured agent, the requested action, the work intent, write permission, and the scope of any requested state change. Results not addressed to the configured agent are normalized to `archive_notification` + `review_only`. Results that request `work_allowed` without `write_permission=state_change_allowed` are normalized back to `review_only`. @@ -794,7 +795,7 @@ Example: | `minConfidence` | number | `0.75` | Minimum classifier confidence required before replacing the deterministic parser action/work-intent result. | | `onlyWhenParserDefaulted` | boolean | `true` | Run the classifier for conservative comment/review classifications and for bot-mentioned comments even when the parser saw implementation-looking language. Set to `false` to let the classifier arbitrate all eligible trusted GitHub comments and reviews. | | `openclawBin` | string | `openclaw` | OpenClaw executable used for classifier subprocess calls. | -| `sessionId` | string | `github-agent-bridge-intent` | Base session id prefix for classifier calls. The bridge derives an isolated per-event session id from this value plus routed agent and GitHub notification/context data. | +| `sessionId` | string | `github-agent-bridge-intent` | Backward-compatible setting retained for older deployments. Current classifier execution uses `openclaw agent exec` and does not rely on a long-lived OpenClaw session id. | | `timeout` | integer | `60` | Maximum seconds to wait for one classifier call before falling back to parser behavior. | The classifier runs before policy decision mapping, but it does not grant trust or bypass authorization. Source trust, `enabledRepos`, `actions`, `trustedRepos`, `trustedOrgs`, routes, roles, and the final `Policy.decision` checks still apply after classification. diff --git a/src/github_agent_bridge/intent_classifier.py b/src/github_agent_bridge/intent_classifier.py index 6b7118b4..edc79460 100644 --- a/src/github_agent_bridge/intent_classifier.py +++ b/src/github_agent_bridge/intent_classifier.py @@ -3,6 +3,7 @@ import json import os import subprocess +import threading from dataclasses import dataclass from typing import Any @@ -24,6 +25,9 @@ ALLOWED_COMPLEXITIES = {"mechanical", "substantive"} INTENT_CLASSIFIER_PROMPT = load_prompt_rule("intent_classifier.md") COMMENT_TARGET_KINDS = {"issue_comment", "review_comment", "commit_comment", "review"} +INTENT_CLASSIFIER_CONCURRENCY = 2 +INTENT_CLASSIFIER_ATTEMPTS = 2 +_INTENT_CLASSIFIER_SEMAPHORE = threading.BoundedSemaphore(INTENT_CLASSIFIER_CONCURRENCY) @dataclass(frozen=True) @@ -199,47 +203,57 @@ def classify_notification_with_llm( agent: str | None = None, prompt_template: str | None = None, ) -> IntentClassification: + prompt = build_intent_prompt(n, ctx, parser_result, prompt_template, policy=policy, agent=agent) cmd = [ cfg.openclaw_bin, "agent", - "--local", + "exec", "--json", - "--session-id", - intent_session_id(cfg.session_id, n, ctx, agent), "--timeout", str(cfg.timeout), "--thinking", cfg.thinking, - "--message", - build_intent_prompt(n, ctx, parser_result, prompt_template, policy=policy, agent=agent), + "--message-file", + "-", ] - if agent: - cmd.extend(["--agent", agent]) if cfg.model: cmd.extend(["--model", cfg.model]) env = os.environ.copy() openclaw_dir = os.path.dirname(cfg.openclaw_bin) if openclaw_dir: env["PATH"] = openclaw_dir + os.pathsep + env.get("PATH", "") - try: - proc = subprocess.run( - cmd, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - timeout=cfg.timeout + 30, - env=env, - ) - except subprocess.TimeoutExpired as exc: - # TimeoutExpired.__str__ contains the complete command and prompt, so - # metadata truncation hides the useful part of the failure. - detail = exc.stderr or exc.stdout or "" - if isinstance(detail, bytes): - detail = detail.decode(errors="replace") - suffix = f": {compact(detail, 300)}" if detail.strip() else "" - raise RuntimeError(f"intent classifier timed out after {cfg.timeout + 30}s{suffix}") from exc - if proc.returncode != 0: - raise RuntimeError(proc.stderr.strip() or proc.stdout.strip() or f"openclaw exited {proc.returncode}") - text = _openclaw_text_from_json(proc.stdout) - return normalize_result(_extract_json_object(text), cfg.min_confidence) + errors: list[str] = [] + with _INTENT_CLASSIFIER_SEMAPHORE: + for attempt in range(1, INTENT_CLASSIFIER_ATTEMPTS + 1): + try: + proc = subprocess.run( + cmd, + input=prompt, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=cfg.timeout + 30, + env=env, + ) + except subprocess.TimeoutExpired as exc: + # TimeoutExpired.__str__ contains the complete command and prompt, so + # metadata truncation hides the useful part of the failure. + detail = exc.stderr or exc.stdout or "" + if isinstance(detail, bytes): + detail = detail.decode(errors="replace") + suffix = f": {compact(detail, 300)}" if detail.strip() else "" + errors.append(f"attempt {attempt}: timed out after {cfg.timeout + 30}s{suffix}") + continue + if proc.returncode != 0: + errors.append( + f"attempt {attempt}: " + + compact(proc.stderr.strip() or proc.stdout.strip() or f"openclaw exited {proc.returncode}", 300) + ) + continue + try: + text = _openclaw_text_from_json(proc.stdout) + return normalize_result(_extract_json_object(text), cfg.min_confidence) + except Exception as exc: + errors.append(f"attempt {attempt}: {compact(str(exc), 300)}") + raise RuntimeError("intent classifier failed after retries: " + " | ".join(errors)) diff --git a/src/github_agent_bridge/queue.py b/src/github_agent_bridge/queue.py index 85b0ae6d..9e464c66 100644 --- a/src/github_agent_bridge/queue.py +++ b/src/github_agent_bridge/queue.py @@ -13,7 +13,7 @@ from .session_correlation import session_id_for_job, session_id_for_job_attempt from . import feedback from .actors import trigger_actor_details_for_enqueue, trigger_actor_details_from_notification -from .intent_classifier import ParserResult, classify_notification_with_llm, should_classify_with_llm +from .intent_classifier import COMMENT_TARGET_KINDS, ParserResult, classify_notification_with_llm, should_classify_with_llm SCHEMA_PACKAGE = "github_agent_bridge.sql" @@ -160,6 +160,12 @@ def ingest( intent = classify_work_intent(n.subject, n.body, policy.bot_logins) metadata: dict[str, object] = {"received_at": n.received_at} parser_result = ParserResult(action, intent) + classifier_required = ( + source == "webhook" + and ctx.target_kind in COMMENT_TARGET_KINDS + and policy.trusted_source(n, ctx) + ) + classifier_applied = False if should_classify_with_llm(n, ctx, parser_result, policy): metadata["intent_classifier"] = { "parser": {"action": action, "work_intent": intent}, @@ -187,11 +193,15 @@ def ingest( if llm_result.applied: action = llm_result.action intent = llm_result.work_intent + classifier_applied = True except Exception as exc: metadata["intent_classifier"] = { **metadata["intent_classifier"], "error": str(exc)[:500], } + if classifier_required and not classifier_applied and intent == "work_allowed": + intent = "review_only" + metadata["intent_guardrail"] = "webhook_classifier_required_read_only" if action == "submit_review": intent = "review_only" metadata["intent_guardrail"] = "submit_review_read_only" diff --git a/tests/test_intent_classifier.py b/tests/test_intent_classifier.py index e819ac09..86589cd1 100644 --- a/tests/test_intent_classifier.py +++ b/tests/test_intent_classifier.py @@ -184,11 +184,11 @@ def test_normalize_result_requires_write_permission_for_work_allowed(): assert result.work_intent == "review_only" -def test_classify_notification_with_llm_uses_isolated_session_id(monkeypatch): +def test_classify_notification_with_llm_uses_agent_exec_with_stdin_prompt(monkeypatch): calls = [] def fake_run(cmd, **kwargs): - calls.append(cmd) + calls.append((cmd, kwargs)) return subprocess.CompletedProcess( cmd, 0, @@ -230,11 +230,10 @@ def fake_run(cmd, **kwargs): prompt_template="Event JSON:\n{event_json}\n", ) - session_ids = [cmd[cmd.index("--session-id") + 1] for cmd in calls] - assert all("--local" in cmd for cmd in calls) - assert session_ids[0].startswith("intent-base-gisce-developer-") - assert session_ids[1].startswith("intent-base-gisce-developer-") - assert session_ids[0] != session_ids[1] + assert [cmd[:4] for cmd, _ in calls] == [["/tmp/openclaw", "agent", "exec", "--json"]] * 2 + assert all("--message-file" in cmd and "-" in cmd for cmd, _ in calls) + assert all("Event JSON:" in kwargs["input"] for _, kwargs in calls) + assert calls[0][1]["input"] != calls[1][1]["input"] def test_classify_notification_with_llm_reports_timeout_without_prompt(monkeypatch): @@ -253,7 +252,53 @@ def fake_run(cmd, **kwargs): IntentClassifier(enabled=True, timeout=60), ) except RuntimeError as exc: - assert str(exc) == "intent classifier timed out after 90s" + assert str(exc) == ( + "intent classifier failed after retries: attempt 1: timed out after 90s | " + "attempt 2: timed out after 90s" + ) assert "Intent classifier prompt" not in str(exc) else: raise AssertionError("expected classifier timeout") + + +def test_classify_notification_with_llm_retries_once(monkeypatch): + notif = notification("<1@github.com>", "@pilipilisbot review this") + calls = 0 + + def fake_run(cmd, **kwargs): + nonlocal calls + calls += 1 + if calls == 1: + return subprocess.CompletedProcess(cmd, 1, "", "temporary failure") + return subprocess.CompletedProcess( + cmd, + 0, + json.dumps({ + "result": { + "payloads": [{ + "text": json.dumps({ + "addressed_to_agent": True, + "action": "reply_comment", + "work_intent": "review_only", + "write_permission": "none", + "confidence": 0.95, + "reason": "Review requested.", + }) + }] + } + }), + "", + ) + + monkeypatch.setattr(subprocess, "run", fake_run) + + result = classify_notification_with_llm( + notif, + extract_github_context(notif.body), + ParserResult("reply_comment", "review_only"), + IntentClassifier(enabled=True, timeout=60), + ) + + assert calls == 2 + assert result.applied is True + assert result.work_intent == "review_only" diff --git a/tests/test_queue.py b/tests/test_queue.py index 6478bfac..9fa2aff2 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -786,6 +786,94 @@ def fake_classify(n, ctx, parser_result, cfg, **kwargs): assert "classifier unavailable" in job.metadata["intent_classifier"]["error"] +def test_webhook_comment_classifier_error_downgrades_parser_work_allowed(tmp_path, monkeypatch): + def fake_classify(n, ctx, parser_result, cfg, **kwargs): + raise RuntimeError("classifier unavailable") + + monkeypatch.setattr("github_agent_bridge.queue.classify_notification_with_llm", fake_classify) + q = JobQueue(tmp_path / "q.sqlite3") + + job, state = q.ingest( + notif( + 1, + "", + "@pilipilisbot implementa això https://github.com/gisce/erp/pull/1#issuecomment-10", + ), + intent_policy(), + source="webhook", + source_key="delivery-1", + ) + + assert state == "enqueued" + assert job.action == "reply_comment" + assert job.metadata["intent_classifier"]["parser"] == {"action": "reply_comment", "work_intent": "work_allowed"} + assert job.work_intent == "review_only" + assert job.metadata["intent_guardrail"] == "webhook_classifier_required_read_only" + + +def test_webhook_comment_low_confidence_downgrades_parser_work_allowed(tmp_path, monkeypatch): + def fake_classify(n, ctx, parser_result, cfg, **kwargs): + return IntentClassification( + action="reply_comment", + work_intent="work_allowed", + confidence=0.4, + reason="Unsure.", + applied=False, + addressed_to_agent=True, + write_permission="state_change_allowed", + ) + + monkeypatch.setattr("github_agent_bridge.queue.classify_notification_with_llm", fake_classify) + q = JobQueue(tmp_path / "q.sqlite3") + + job, state = q.ingest( + notif( + 1, + "", + "@pilipilisbot implementa això https://github.com/gisce/erp/pull/1#issuecomment-10", + ), + intent_policy(min_confidence=0.75), + source="webhook", + source_key="delivery-2", + ) + + assert state == "enqueued" + assert job.work_intent == "review_only" + assert job.metadata["intent_classifier"]["llm"]["applied"] is False + assert job.metadata["intent_guardrail"] == "webhook_classifier_required_read_only" + + +def test_webhook_comment_high_confidence_classifier_can_allow_work(tmp_path, monkeypatch): + def fake_classify(n, ctx, parser_result, cfg, **kwargs): + return IntentClassification( + action="reply_comment", + work_intent="work_allowed", + confidence=0.91, + reason="User asks for implementation.", + applied=True, + addressed_to_agent=True, + write_permission="state_change_allowed", + ) + + monkeypatch.setattr("github_agent_bridge.queue.classify_notification_with_llm", fake_classify) + q = JobQueue(tmp_path / "q.sqlite3") + + job, state = q.ingest( + notif( + 1, + "", + "@pilipilisbot implementa això https://github.com/gisce/erp/pull/1#issuecomment-10", + ), + intent_policy(min_confidence=0.75), + source="webhook", + source_key="delivery-3", + ) + + assert state == "enqueued" + assert job.work_intent == "work_allowed" + assert job.metadata["intent_classifier"]["llm"]["applied"] is True + + def test_enqueue_skips_llm_intent_classifier_when_disabled(tmp_path, monkeypatch): calls = [] monkeypatch.setattr("github_agent_bridge.queue.classify_notification_with_llm", lambda *args, **kwargs: calls.append(args) or None) diff --git a/tests/test_webhook.py b/tests/test_webhook.py index f84efa31..3e8bff90 100644 --- a/tests/test_webhook.py +++ b/tests/test_webhook.py @@ -288,7 +288,7 @@ def test_webhook_canary_enqueues_enabled_actionable_repository_once(tmp_path): "status": "pending", "action": "reply_comment", "decision": "auto_trusted", - "work_intent": "work_allowed", + "work_intent": "review_only", "updated_at": detail["job"]["updated_at"], } with sqlite3.connect(config.db) as con: From 5faf4f7da45c8260af8144cf557f986d03f62c04 Mon Sep 17 00:00:00 2001 From: Pilipilis Date: Thu, 8 Oct 2026 21:08:32 +0000 Subject: [PATCH 2/5] fix: verify OpenClaw agent exec contract Co-authored-by: giscebot --- docs/installation.md | 2 +- docs/operations.md | 11 ++-- docs/policy-reference.md | 11 ++-- src/github_agent_bridge/intent_classifier.py | 34 +++++++++++ tests/test_intent_classifier.py | 59 ++++++++++++++++++-- 5 files changed, 103 insertions(+), 14 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 4acf98bc..d2d9416e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -7,7 +7,7 @@ This guide shows how to install `github-agent-bridge` for a new OpenClaw deploym Before installing the bridge, have these ready: - Python 3.11 or newer. -- OpenClaw CLI installed and able to run `openclaw agent`. +- OpenClaw CLI installed with `openclaw agent exec` support for the intent classifier. - GitHub CLI (`gh`) installed and authenticated as the GitHub user/bot that should react to comments. - An email inbox that receives GitHub notification emails. - IMAP access to that inbox, usually an app password. diff --git a/docs/operations.md b/docs/operations.md index 335b44a4..d032a7c1 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -80,10 +80,13 @@ starve other gateway work. The enqueue-time intent classifier is deliberately different: the bridge calls `openclaw agent exec` for a one-shot classification. This avoids depending on -the long-lived Gateway session path for the permission decision. The bridge -limits classifier subprocess concurrency, retries once, and treats webhook -comment/review classifier failure as `review_only`, so a classifier outage cannot -elevate webhook work to `work_allowed`. +the long-lived Gateway session path for the permission decision. The OpenClaw +CLI on classifier hosts must expose `openclaw agent exec --message-file --json +--timeout --thinking`; the bridge checks that contract before invoking the +classifier. The bridge limits classifier subprocess concurrency, retries once, +and treats webhook comment/review classifier failure as `review_only`, so a +classifier outage or unsupported CLI cannot elevate webhook work to +`work_allowed`. For the standard four-worker deployment, use eight OpenClaw slots: diff --git a/docs/policy-reference.md b/docs/policy-reference.md index 2543d1b4..c859eef8 100644 --- a/docs/policy-reference.md +++ b/docs/policy-reference.md @@ -770,10 +770,13 @@ Agents must also apply the comment value rule before posting: comment only when `intentClassifier` controls an optional enqueue-time LLM classifier for trusted GitHub comments and reviews. It is disabled by default; when enabled, the bridge calls OpenClaw with the packaged `prompt_rules/intent_classifier.md` prompt or `promptOverrides.rules.intent_classifier`, expects JSON output, and uses the result only when confidence is high enough. Low-confidence, invalid, timed-out, or failed webhook comment/review classifier calls fall back to `review_only`; email ingestion keeps the deterministic parser result for backward compatibility. Classifier calls use `openclaw agent exec`, avoiding the long-lived Gateway -session path for enqueue-time routing. The bridge retries once, limits classifier -subprocess concurrency, and treats failure as `review_only` for webhook -comments/reviews. Normal executor, feedback-learning, and interactive gateway -calls still need suitable OpenClaw concurrency headroom; see +session path for enqueue-time routing. The bridge verifies the +`openclaw agent exec --help` contract before classification, retries once, +limits classifier subprocess concurrency, and treats failure as `review_only` +for webhook comments/reviews. Deployments that enable the classifier must run an +OpenClaw CLI build that exposes `agent exec`, `--message-file`, `--json`, +`--timeout`, and `--thinking`. Normal executor, feedback-learning, and +interactive gateway calls still need suitable OpenClaw concurrency headroom; see [`operations.md`](operations.md#openclaw-concurrency-headroom). The classifier returns structured semantics: whether the event is addressed to the configured agent, the requested action, the work intent, write permission, and the scope of any requested state change. Results not addressed to the configured agent are normalized to `archive_notification` + `review_only`. Results that request `work_allowed` without `write_permission=state_change_allowed` are normalized back to `review_only`. diff --git a/src/github_agent_bridge/intent_classifier.py b/src/github_agent_bridge/intent_classifier.py index 74fbad5f..f997ad68 100644 --- a/src/github_agent_bridge/intent_classifier.py +++ b/src/github_agent_bridge/intent_classifier.py @@ -6,6 +6,7 @@ import subprocess import threading from dataclasses import dataclass +from functools import lru_cache from typing import Any from .feedback import _extract_json_object, _openclaw_text_from_json, compact, load_prompt_rule, session_id_for_event @@ -79,6 +80,38 @@ def to_metadata(self) -> dict[str, object]: return metadata +@lru_cache(maxsize=8) +def assert_openclaw_agent_exec_supported(openclaw_bin: str) -> None: + try: + proc = subprocess.run( + [openclaw_bin, "agent", "exec", "--help"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + detail = compact(str(exc), 300) + raise RuntimeError( + f"OpenClaw CLI does not support required 'agent exec' contract: {detail}" + ) + + output = proc.stdout + proc.stderr + required = ( + "Usage: openclaw agent exec", + "--message-file", + "--json", + "--timeout", + "--thinking", + ) + if proc.returncode != 0 or not all(token in output for token in required): + detail = compact(output.strip() or f"openclaw exited {proc.returncode}", 300) + raise RuntimeError( + f"OpenClaw CLI does not support required 'agent exec' contract: {detail}" + ) + + def should_classify_with_llm( n: Notification, ctx: GitHubContext, @@ -219,6 +252,7 @@ def classify_notification_with_llm( agent: str | None = None, prompt_template: str | None = None, ) -> IntentClassification: + assert_openclaw_agent_exec_supported(cfg.openclaw_bin) prompt = build_intent_prompt(n, ctx, parser_result, prompt_template, policy=policy, agent=agent) cmd = [ cfg.openclaw_bin, diff --git a/tests/test_intent_classifier.py b/tests/test_intent_classifier.py index 70b896b7..4886be26 100644 --- a/tests/test_intent_classifier.py +++ b/tests/test_intent_classifier.py @@ -1,9 +1,11 @@ import json +import pytest import shutil import subprocess from github_agent_bridge.intent_classifier import ( ParserResult, + assert_openclaw_agent_exec_supported, build_intent_prompt, classify_notification_with_llm, intent_session_id, @@ -215,9 +217,17 @@ def test_normalize_result_requires_write_permission_for_work_allowed(): def test_classify_notification_with_llm_uses_agent_exec_with_stdin_prompt(monkeypatch): calls = [] + assert_openclaw_agent_exec_supported.cache_clear() def fake_run(cmd, **kwargs): calls.append((cmd, kwargs)) + if cmd[-1] == "--help": + return subprocess.CompletedProcess( + cmd, + 0, + "Usage: openclaw agent exec\n--message-file \n--json\n--timeout \n--thinking \n", + "", + ) return subprocess.CompletedProcess( cmd, 0, @@ -259,16 +269,39 @@ def fake_run(cmd, **kwargs): prompt_template="Event JSON:\n{event_json}\n", ) - assert [cmd[:4] for cmd, _ in calls] == [["/tmp/openclaw", "agent", "exec", "--json"]] * 2 - assert all("--message-file" in cmd and "-" in cmd for cmd, _ in calls) - assert all("Event JSON:" in kwargs["input"] for _, kwargs in calls) - assert calls[0][1]["input"] != calls[1][1]["input"] + classifier_calls = [(cmd, kwargs) for cmd, kwargs in calls if cmd[-1] != "--help"] + assert [cmd[:4] for cmd, _ in classifier_calls] == [["/tmp/openclaw", "agent", "exec", "--json"]] * 2 + assert all("--message-file" in cmd and "-" in cmd for cmd, _ in classifier_calls) + assert all("Event JSON:" in kwargs["input"] for _, kwargs in classifier_calls) + assert classifier_calls[0][1]["input"] != classifier_calls[1][1]["input"] + + +def test_openclaw_agent_exec_contract_rejects_unsupported_cli(monkeypatch): + assert_openclaw_agent_exec_supported.cache_clear() + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess( + cmd, + 0, + "Usage: openclaw agent [options] [command]\nCommands:\n run\n", + "", + ) + + monkeypatch.setattr(subprocess, "run", fake_run) + + try: + assert_openclaw_agent_exec_supported("/tmp/openclaw") + except RuntimeError as exc: + assert "required 'agent exec' contract" in str(exc) + assert "Usage: openclaw agent [options] [command]" in str(exc) + else: + raise AssertionError("expected unsupported OpenClaw CLI to fail the contract check") def test_openclaw_agent_exec_cli_accepts_classifier_contract(): openclaw_bin = shutil.which("openclaw") if not openclaw_bin: - return + pytest.skip("openclaw CLI is not installed") proc = subprocess.run( [ @@ -295,8 +328,16 @@ def test_openclaw_agent_exec_cli_accepts_classifier_contract(): def test_classify_notification_with_llm_reports_timeout_without_prompt(monkeypatch): notif = notification("<1@github.com>", "@pilipilisbot review this") + assert_openclaw_agent_exec_supported.cache_clear() def fake_run(cmd, **kwargs): + if cmd[-1] == "--help": + return subprocess.CompletedProcess( + cmd, + 0, + "Usage: openclaw agent exec\n--message-file \n--json\n--timeout \n--thinking \n", + "", + ) raise subprocess.TimeoutExpired(cmd, kwargs["timeout"]) monkeypatch.setattr(subprocess, "run", fake_run) @@ -321,9 +362,17 @@ def fake_run(cmd, **kwargs): def test_classify_notification_with_llm_retries_once(monkeypatch): notif = notification("<1@github.com>", "@pilipilisbot review this") calls = 0 + assert_openclaw_agent_exec_supported.cache_clear() def fake_run(cmd, **kwargs): nonlocal calls + if cmd[-1] == "--help": + return subprocess.CompletedProcess( + cmd, + 0, + "Usage: openclaw agent exec\n--message-file \n--json\n--timeout \n--thinking \n", + "", + ) calls += 1 if calls == 1: return subprocess.CompletedProcess(cmd, 1, "", "temporary failure") From 6f1f1107e2dbd01cd8e70e4c56e6695462a75df0 Mon Sep 17 00:00:00 2001 From: Pilipilis Date: Thu, 8 Oct 2026 21:33:56 +0000 Subject: [PATCH 3/5] fix: run intent classifier without coding tools Force enqueue-time intent classification through OpenClaw agent exec direct mode, document the required CLI contract, and cover the disabled-classifier webhook fallback. Co-authored-by: giscebot --- docs/installation.md | 2 +- docs/operations.md | 14 +++++++----- docs/policy-reference.md | 24 +++++++++++--------- src/github_agent_bridge/intent_classifier.py | 4 ++++ tests/test_intent_classifier.py | 19 +++++++++++++--- tests/test_queue.py | 22 ++++++++++++++++++ 6 files changed, 64 insertions(+), 21 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index d2d9416e..66bbba21 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -7,7 +7,7 @@ This guide shows how to install `github-agent-bridge` for a new OpenClaw deploym Before installing the bridge, have these ready: - Python 3.11 or newer. -- OpenClaw CLI installed with `openclaw agent exec` support for the intent classifier. +- OpenClaw CLI 2026.9.9 or newer installed with `openclaw agent exec --code-mode direct` support for the intent classifier. - GitHub CLI (`gh`) installed and authenticated as the GitHub user/bot that should react to comments. - An email inbox that receives GitHub notification emails. - IMAP access to that inbox, usually an app password. diff --git a/docs/operations.md b/docs/operations.md index d032a7c1..d633eee7 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -81,12 +81,14 @@ starve other gateway work. The enqueue-time intent classifier is deliberately different: the bridge calls `openclaw agent exec` for a one-shot classification. This avoids depending on the long-lived Gateway session path for the permission decision. The OpenClaw -CLI on classifier hosts must expose `openclaw agent exec --message-file --json ---timeout --thinking`; the bridge checks that contract before invoking the -classifier. The bridge limits classifier subprocess concurrency, retries once, -and treats webhook comment/review classifier failure as `review_only`, so a -classifier outage or unsupported CLI cannot elevate webhook work to -`work_allowed`. +CLI on classifier hosts must be 2026.9.9 or newer and expose `openclaw agent +exec --code-mode direct --message-file --json --timeout --thinking`; the bridge +checks that contract before invoking the classifier. Classifier subprocesses run +with `--code-mode direct` so untrusted GitHub text is interpreted as a +classification prompt, not as a coding-agent run with repository tools. The +bridge limits classifier subprocess concurrency, retries once, and treats +webhook comment/review classifier failure as `review_only`, so a classifier +outage or unsupported CLI cannot elevate webhook work to `work_allowed`. For the standard four-worker deployment, use eight OpenClaw slots: diff --git a/docs/policy-reference.md b/docs/policy-reference.md index c859eef8..e8744ad8 100644 --- a/docs/policy-reference.md +++ b/docs/policy-reference.md @@ -767,16 +767,18 @@ Agents must also apply the comment value rule before posting: comment only when ## Intent classifier -`intentClassifier` controls an optional enqueue-time LLM classifier for trusted GitHub comments and reviews. It is disabled by default; when enabled, the bridge calls OpenClaw with the packaged `prompt_rules/intent_classifier.md` prompt or `promptOverrides.rules.intent_classifier`, expects JSON output, and uses the result only when confidence is high enough. Low-confidence, invalid, timed-out, or failed webhook comment/review classifier calls fall back to `review_only`; email ingestion keeps the deterministic parser result for backward compatibility. - -Classifier calls use `openclaw agent exec`, avoiding the long-lived Gateway -session path for enqueue-time routing. The bridge verifies the -`openclaw agent exec --help` contract before classification, retries once, -limits classifier subprocess concurrency, and treats failure as `review_only` -for webhook comments/reviews. Deployments that enable the classifier must run an -OpenClaw CLI build that exposes `agent exec`, `--message-file`, `--json`, -`--timeout`, and `--thinking`. Normal executor, feedback-learning, and -interactive gateway calls still need suitable OpenClaw concurrency headroom; see +`intentClassifier` controls an optional enqueue-time LLM classifier for trusted GitHub comments and reviews. It is disabled by default; when enabled, the bridge calls OpenClaw with the packaged `prompt_rules/intent_classifier.md` prompt or `promptOverrides.rules.intent_classifier`, expects JSON output, and uses the result only when confidence is high enough. Webhook comment/review parser results are still guarded even when the classifier is disabled: only a high-confidence applied classifier result, or the explicit structured `pr_authored_by_bot` feedback path, may keep/elevate webhook comment work to `work_allowed`. Low-confidence, invalid, timed-out, failed, or disabled webhook comment/review classifier paths fall back to `review_only`; email ingestion keeps the deterministic parser result for backward compatibility. + +Classifier calls use `openclaw agent exec --code-mode direct`, avoiding the +long-lived Gateway session path for enqueue-time routing and preventing +untrusted GitHub text from running as a coding-agent turn with repository tools. +The bridge verifies the `openclaw agent exec --help` contract before +classification, retries once, limits classifier subprocess concurrency, and +treats failure as `review_only` for webhook comments/reviews. Deployments that +enable the classifier must run OpenClaw CLI 2026.9.9 or newer with `agent exec`, +`--code-mode`, `--message-file`, `--json`, `--timeout`, and `--thinking`. +Normal executor, feedback-learning, and interactive gateway calls still need +suitable OpenClaw concurrency headroom; see [`operations.md`](operations.md#openclaw-concurrency-headroom). The classifier returns structured semantics: whether the event is addressed to the configured agent, the requested action, the work intent, write permission, and the scope of any requested state change. Results not addressed to the configured agent are normalized to `archive_notification` + `review_only`. Results that request `work_allowed` without `write_permission=state_change_allowed` are normalized back to `review_only`. @@ -807,7 +809,7 @@ Example: | `onlyWhenParserDefaulted` | boolean | `true` | Run the classifier for conservative comment/review classifications and for bot-mentioned comments even when the parser saw implementation-looking language. Set to `false` to let the classifier arbitrate all eligible trusted GitHub comments and reviews. | | `openclawBin` | string | `openclaw` | OpenClaw executable used for classifier subprocess calls. | | `sessionId` | string | `github-agent-bridge-intent` | Backward-compatible setting retained for older deployments. Current classifier execution uses `openclaw agent exec` and does not rely on a long-lived OpenClaw session id. | -| `timeout` | integer | `60` | Maximum seconds to wait for one classifier call before falling back to parser behavior. | +| `timeout` | integer | `60` | Maximum seconds to wait for one classifier call before falling back conservatively for webhook comments/reviews. | The classifier runs before policy decision mapping, but it does not grant trust or bypass authorization. Source trust, `enabledRepos`, `actions`, `trustedRepos`, `trustedOrgs`, routes, roles, and the final `Policy.decision` checks still apply after classification. diff --git a/src/github_agent_bridge/intent_classifier.py b/src/github_agent_bridge/intent_classifier.py index f997ad68..befb0ff1 100644 --- a/src/github_agent_bridge/intent_classifier.py +++ b/src/github_agent_bridge/intent_classifier.py @@ -100,6 +100,8 @@ def assert_openclaw_agent_exec_supported(openclaw_bin: str) -> None: output = proc.stdout + proc.stderr required = ( "Usage: openclaw agent exec", + "--code-mode", + "direct", "--message-file", "--json", "--timeout", @@ -259,6 +261,8 @@ def classify_notification_with_llm( "agent", "exec", "--json", + "--code-mode", + "direct", "--timeout", str(cfg.timeout), "--thinking", diff --git a/tests/test_intent_classifier.py b/tests/test_intent_classifier.py index 4886be26..e8fede51 100644 --- a/tests/test_intent_classifier.py +++ b/tests/test_intent_classifier.py @@ -225,7 +225,10 @@ def fake_run(cmd, **kwargs): return subprocess.CompletedProcess( cmd, 0, - "Usage: openclaw agent exec\n--message-file \n--json\n--timeout \n--thinking \n", + "Usage: openclaw agent exec\n" + "--code-mode direct | auto | code\n" + "--message-file \n--json\n--timeout \n" + "--thinking \n", "", ) return subprocess.CompletedProcess( @@ -271,6 +274,8 @@ def fake_run(cmd, **kwargs): classifier_calls = [(cmd, kwargs) for cmd, kwargs in calls if cmd[-1] != "--help"] assert [cmd[:4] for cmd, _ in classifier_calls] == [["/tmp/openclaw", "agent", "exec", "--json"]] * 2 + assert all("--code-mode" in cmd for cmd, _ in classifier_calls) + assert all(cmd[cmd.index("--code-mode") + 1] == "direct" for cmd, _ in classifier_calls) assert all("--message-file" in cmd and "-" in cmd for cmd, _ in classifier_calls) assert all("Event JSON:" in kwargs["input"] for _, kwargs in classifier_calls) assert classifier_calls[0][1]["input"] != classifier_calls[1][1]["input"] @@ -320,6 +325,8 @@ def test_openclaw_agent_exec_cli_accepts_classifier_contract(): output = proc.stdout + proc.stderr assert proc.returncode == 0 assert "Usage: openclaw agent exec" in output + assert "--code-mode " in output + assert "direct" in output assert "--message-file " in output assert "--json" in output assert "--timeout " in output @@ -335,7 +342,10 @@ def fake_run(cmd, **kwargs): return subprocess.CompletedProcess( cmd, 0, - "Usage: openclaw agent exec\n--message-file \n--json\n--timeout \n--thinking \n", + "Usage: openclaw agent exec\n" + "--code-mode direct | auto | code\n" + "--message-file \n--json\n--timeout \n" + "--thinking \n", "", ) raise subprocess.TimeoutExpired(cmd, kwargs["timeout"]) @@ -370,7 +380,10 @@ def fake_run(cmd, **kwargs): return subprocess.CompletedProcess( cmd, 0, - "Usage: openclaw agent exec\n--message-file \n--json\n--timeout \n--thinking \n", + "Usage: openclaw agent exec\n" + "--code-mode direct | auto | code\n" + "--message-file \n--json\n--timeout \n" + "--thinking \n", "", ) calls += 1 diff --git a/tests/test_queue.py b/tests/test_queue.py index c5b7a23f..6722c4ca 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -1175,6 +1175,28 @@ def test_enqueue_skips_llm_intent_classifier_when_disabled(tmp_path, monkeypatch assert calls == [] +def test_webhook_comment_classifier_disabled_downgrades_parser_work_allowed(tmp_path, monkeypatch): + calls = [] + monkeypatch.setattr("github_agent_bridge.queue.classify_notification_with_llm", lambda *args, **kwargs: calls.append(args) or None) + q = JobQueue(tmp_path / "q.sqlite3") + + job, state = q.ingest( + notif( + 1, + "", + "@pilipilisbot implementa això https://github.com/gisce/erp/pull/1#issuecomment-10", + ), + policy(), + source="webhook", + source_key="delivery-disabled", + ) + + assert state == "enqueued" + assert calls == [] + assert job.work_intent == "review_only" + assert job.metadata["intent_guardrail"] == "webhook_classifier_required_read_only" + + def test_enqueue_merge_issue_event_forces_review_only(tmp_path, monkeypatch): monkeypatch.setattr( "github_agent_bridge.actors.github_actor_details_for_context", From 915a6e9a9ba0c07f2a06ad6eaed92a65028d38e8 Mon Sep 17 00:00:00 2001 From: Pilipilis Date: Fri, 9 Oct 2026 09:08:26 +0000 Subject: [PATCH 4/5] fix: use tool-less intent classifier inference Run enqueue-time intent classification through OpenClaw raw model inference instead of agent exec, so untrusted GitHub text is not processed by a coding-agent tool surface before work_intent is decided. Co-authored-by: giscebot --- docs/installation.md | 2 +- docs/operations.md | 20 ++--- docs/policy-reference.md | 19 +++-- src/github_agent_bridge/intent_classifier.py | 78 ++++++++++++----- tests/test_intent_classifier.py | 88 ++++++++++---------- 5 files changed, 122 insertions(+), 85 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 66bbba21..d627ca9e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -7,7 +7,7 @@ This guide shows how to install `github-agent-bridge` for a new OpenClaw deploym Before installing the bridge, have these ready: - Python 3.11 or newer. -- OpenClaw CLI 2026.9.9 or newer installed with `openclaw agent exec --code-mode direct` support for the intent classifier. +- OpenClaw CLI installed with `openclaw infer model run --local --prompt --json` support for the intent classifier. - GitHub CLI (`gh`) installed and authenticated as the GitHub user/bot that should react to comments. - An email inbox that receives GitHub notification emails. - IMAP access to that inbox, usually an app password. diff --git a/docs/operations.md b/docs/operations.md index d633eee7..89119d20 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -79,16 +79,16 @@ count. Otherwise four busy workers can fill a four-slot OpenClaw queue and starve other gateway work. The enqueue-time intent classifier is deliberately different: the bridge calls -`openclaw agent exec` for a one-shot classification. This avoids depending on -the long-lived Gateway session path for the permission decision. The OpenClaw -CLI on classifier hosts must be 2026.9.9 or newer and expose `openclaw agent -exec --code-mode direct --message-file --json --timeout --thinking`; the bridge -checks that contract before invoking the classifier. Classifier subprocesses run -with `--code-mode direct` so untrusted GitHub text is interpreted as a -classification prompt, not as a coding-agent run with repository tools. The -bridge limits classifier subprocess concurrency, retries once, and treats -webhook comment/review classifier failure as `review_only`, so a classifier -outage or unsupported CLI cannot elevate webhook work to `work_allowed`. +`openclaw infer model run --local --prompt --json` for a one-shot raw model +classification. This avoids both the long-lived Gateway session path and the +`agent exec` coding-agent tool surface for the permission decision. The OpenClaw +CLI on classifier hosts must expose `openclaw infer model run` with `--local`, +`--prompt`, `--json`, `--model`, and `--thinking`; the bridge checks that +contract before invoking the classifier. The raw model inference path does not +load repository tools, bundled MCP servers, or session transcript. The bridge +limits classifier subprocess concurrency, retries once, and treats webhook +comment/review classifier failure as `review_only`, so a classifier outage or +unsupported CLI cannot elevate webhook work to `work_allowed`. For the standard four-worker deployment, use eight OpenClaw slots: diff --git a/docs/policy-reference.md b/docs/policy-reference.md index e8744ad8..435a891e 100644 --- a/docs/policy-reference.md +++ b/docs/policy-reference.md @@ -769,14 +769,15 @@ Agents must also apply the comment value rule before posting: comment only when `intentClassifier` controls an optional enqueue-time LLM classifier for trusted GitHub comments and reviews. It is disabled by default; when enabled, the bridge calls OpenClaw with the packaged `prompt_rules/intent_classifier.md` prompt or `promptOverrides.rules.intent_classifier`, expects JSON output, and uses the result only when confidence is high enough. Webhook comment/review parser results are still guarded even when the classifier is disabled: only a high-confidence applied classifier result, or the explicit structured `pr_authored_by_bot` feedback path, may keep/elevate webhook comment work to `work_allowed`. Low-confidence, invalid, timed-out, failed, or disabled webhook comment/review classifier paths fall back to `review_only`; email ingestion keeps the deterministic parser result for backward compatibility. -Classifier calls use `openclaw agent exec --code-mode direct`, avoiding the -long-lived Gateway session path for enqueue-time routing and preventing -untrusted GitHub text from running as a coding-agent turn with repository tools. -The bridge verifies the `openclaw agent exec --help` contract before -classification, retries once, limits classifier subprocess concurrency, and -treats failure as `review_only` for webhook comments/reviews. Deployments that -enable the classifier must run OpenClaw CLI 2026.9.9 or newer with `agent exec`, -`--code-mode`, `--message-file`, `--json`, `--timeout`, and `--thinking`. +Classifier calls use `openclaw infer model run --local --prompt --json`, not +`openclaw agent exec`. This uses OpenClaw's raw model inference path, which does +not start a coding-agent turn, load repository tools, load bundled MCP servers, +or include prior session transcript. The bridge verifies the `openclaw infer +model run --help` contract before classification, retries once, limits +classifier subprocess concurrency, and treats failure as `review_only` for +webhook comments/reviews. Deployments that enable the classifier must run an +OpenClaw CLI that exposes `infer model run`, `--local`, `--prompt`, `--json`, +`--model`, and `--thinking`. Normal executor, feedback-learning, and interactive gateway calls still need suitable OpenClaw concurrency headroom; see [`operations.md`](operations.md#openclaw-concurrency-headroom). @@ -808,7 +809,7 @@ Example: | `minConfidence` | number | `0.75` | Minimum classifier confidence required before replacing the deterministic parser action/work-intent result. | | `onlyWhenParserDefaulted` | boolean | `true` | Run the classifier for conservative comment/review classifications and for bot-mentioned comments even when the parser saw implementation-looking language. Set to `false` to let the classifier arbitrate all eligible trusted GitHub comments and reviews. | | `openclawBin` | string | `openclaw` | OpenClaw executable used for classifier subprocess calls. | -| `sessionId` | string | `github-agent-bridge-intent` | Backward-compatible setting retained for older deployments. Current classifier execution uses `openclaw agent exec` and does not rely on a long-lived OpenClaw session id. | +| `sessionId` | string | `github-agent-bridge-intent` | Backward-compatible setting retained for older deployments. Current classifier execution uses raw `openclaw infer model run` and does not rely on a long-lived OpenClaw session id. | | `timeout` | integer | `60` | Maximum seconds to wait for one classifier call before falling back conservatively for webhook comments/reviews. | The classifier runs before policy decision mapping, but it does not grant trust or bypass authorization. Source trust, `enabledRepos`, `actions`, `trustedRepos`, `trustedOrgs`, routes, roles, and the final `Policy.decision` checks still apply after classification. diff --git a/src/github_agent_bridge/intent_classifier.py b/src/github_agent_bridge/intent_classifier.py index befb0ff1..d4e4a0eb 100644 --- a/src/github_agent_bridge/intent_classifier.py +++ b/src/github_agent_bridge/intent_classifier.py @@ -9,7 +9,7 @@ from functools import lru_cache from typing import Any -from .feedback import _extract_json_object, _openclaw_text_from_json, compact, load_prompt_rule, session_id_for_event +from .feedback import _extract_json_object, compact, load_prompt_rule, session_id_for_event from .models import GitHubContext, Notification from .parser import github_event_flags from .policy import DEFAULT_BOT_LOGINS, IntentClassifier, Policy @@ -80,37 +80,74 @@ def to_metadata(self) -> dict[str, object]: return metadata +def _text_from_model_run_json(raw: str) -> str: + data = json.loads(raw) + if isinstance(data, str): + return data + if not isinstance(data, dict): + return raw + candidates = [ + data.get("text"), + data.get("output_text"), + data.get("output"), + data.get("content"), + data.get("message"), + data.get("response"), + ] + result = data.get("result") + if isinstance(result, dict): + candidates.extend( + [ + result.get("text"), + result.get("output_text"), + result.get("output"), + result.get("content"), + result.get("message"), + result.get("response"), + ] + ) + payloads = result.get("payloads") + if isinstance(payloads, list): + for payload in payloads: + if isinstance(payload, dict): + candidates.append(payload.get("text")) + candidates.append(payload.get("content")) + for candidate in candidates: + if isinstance(candidate, str) and candidate.strip(): + return candidate + return raw + + @lru_cache(maxsize=8) -def assert_openclaw_agent_exec_supported(openclaw_bin: str) -> None: +def assert_openclaw_model_run_supported(openclaw_bin: str) -> None: try: proc = subprocess.run( - [openclaw_bin, "agent", "exec", "--help"], + [openclaw_bin, "infer", "model", "run", "--help"], check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=10, - ) + ) except (OSError, subprocess.TimeoutExpired) as exc: detail = compact(str(exc), 300) raise RuntimeError( - f"OpenClaw CLI does not support required 'agent exec' contract: {detail}" + f"OpenClaw CLI does not support required tool-less model run contract: {detail}" ) output = proc.stdout + proc.stderr required = ( - "Usage: openclaw agent exec", - "--code-mode", - "direct", - "--message-file", + "Usage: openclaw infer model run", + "--local", + "--prompt", "--json", - "--timeout", + "--model", "--thinking", ) if proc.returncode != 0 or not all(token in output for token in required): detail = compact(output.strip() or f"openclaw exited {proc.returncode}", 300) raise RuntimeError( - f"OpenClaw CLI does not support required 'agent exec' contract: {detail}" + f"OpenClaw CLI does not support required tool-less model run contract: {detail}" ) @@ -254,21 +291,19 @@ def classify_notification_with_llm( agent: str | None = None, prompt_template: str | None = None, ) -> IntentClassification: - assert_openclaw_agent_exec_supported(cfg.openclaw_bin) + assert_openclaw_model_run_supported(cfg.openclaw_bin) prompt = build_intent_prompt(n, ctx, parser_result, prompt_template, policy=policy, agent=agent) cmd = [ cfg.openclaw_bin, - "agent", - "exec", + "infer", + "model", + "run", + "--local", "--json", - "--code-mode", - "direct", - "--timeout", - str(cfg.timeout), + "--prompt", + prompt, "--thinking", cfg.thinking, - "--message-file", - "-", ] if cfg.model: cmd.extend(["--model", cfg.model]) @@ -282,7 +317,6 @@ def classify_notification_with_llm( try: proc = subprocess.run( cmd, - input=prompt, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -306,7 +340,7 @@ def classify_notification_with_llm( ) continue try: - text = _openclaw_text_from_json(proc.stdout) + text = _text_from_model_run_json(proc.stdout) return normalize_result(_extract_json_object(text), cfg.min_confidence) except Exception as exc: errors.append(f"attempt {attempt}: {compact(str(exc), 300)}") diff --git a/tests/test_intent_classifier.py b/tests/test_intent_classifier.py index e8fede51..75e81f31 100644 --- a/tests/test_intent_classifier.py +++ b/tests/test_intent_classifier.py @@ -5,7 +5,7 @@ from github_agent_bridge.intent_classifier import ( ParserResult, - assert_openclaw_agent_exec_supported, + assert_openclaw_model_run_supported, build_intent_prompt, classify_notification_with_llm, intent_session_id, @@ -215,9 +215,9 @@ def test_normalize_result_requires_write_permission_for_work_allowed(): assert result.work_intent == "review_only" -def test_classify_notification_with_llm_uses_agent_exec_with_stdin_prompt(monkeypatch): +def test_classify_notification_with_llm_uses_toolless_model_run(monkeypatch): calls = [] - assert_openclaw_agent_exec_supported.cache_clear() + assert_openclaw_model_run_supported.cache_clear() def fake_run(cmd, **kwargs): calls.append((cmd, kwargs)) @@ -225,9 +225,8 @@ def fake_run(cmd, **kwargs): return subprocess.CompletedProcess( cmd, 0, - "Usage: openclaw agent exec\n" - "--code-mode direct | auto | code\n" - "--message-file \n--json\n--timeout \n" + "Usage: openclaw infer model run\n" + "--local\n--prompt \n--json\n--model \n" "--thinking \n", "", ) @@ -273,16 +272,18 @@ def fake_run(cmd, **kwargs): ) classifier_calls = [(cmd, kwargs) for cmd, kwargs in calls if cmd[-1] != "--help"] - assert [cmd[:4] for cmd, _ in classifier_calls] == [["/tmp/openclaw", "agent", "exec", "--json"]] * 2 - assert all("--code-mode" in cmd for cmd, _ in classifier_calls) - assert all(cmd[cmd.index("--code-mode") + 1] == "direct" for cmd, _ in classifier_calls) - assert all("--message-file" in cmd and "-" in cmd for cmd, _ in classifier_calls) - assert all("Event JSON:" in kwargs["input"] for _, kwargs in classifier_calls) - assert classifier_calls[0][1]["input"] != classifier_calls[1][1]["input"] + assert [cmd[:5] for cmd, _ in classifier_calls] == [["/tmp/openclaw", "infer", "model", "run", "--local"]] * 2 + assert all("--prompt" in cmd for cmd, _ in classifier_calls) + assert all("Event JSON:" in cmd[cmd.index("--prompt") + 1] for cmd, _ in classifier_calls) + assert all("--code-mode" not in cmd and "agent" not in cmd for cmd, _ in classifier_calls) + assert all("input" not in kwargs for _, kwargs in classifier_calls) + assert classifier_calls[0][0][classifier_calls[0][0].index("--prompt") + 1] != classifier_calls[1][0][ + classifier_calls[1][0].index("--prompt") + 1 + ] -def test_openclaw_agent_exec_contract_rejects_unsupported_cli(monkeypatch): - assert_openclaw_agent_exec_supported.cache_clear() +def test_openclaw_model_run_contract_rejects_unsupported_cli(monkeypatch): + assert_openclaw_model_run_supported.cache_clear() def fake_run(cmd, **kwargs): return subprocess.CompletedProcess( @@ -295,56 +296,58 @@ def fake_run(cmd, **kwargs): monkeypatch.setattr(subprocess, "run", fake_run) try: - assert_openclaw_agent_exec_supported("/tmp/openclaw") + assert_openclaw_model_run_supported("/tmp/openclaw") except RuntimeError as exc: - assert "required 'agent exec' contract" in str(exc) + assert "required tool-less model run contract" in str(exc) assert "Usage: openclaw agent [options] [command]" in str(exc) else: raise AssertionError("expected unsupported OpenClaw CLI to fail the contract check") -def test_openclaw_agent_exec_cli_accepts_classifier_contract(): +def test_openclaw_model_run_cli_accepts_classifier_contract(): openclaw_bin = shutil.which("openclaw") if not openclaw_bin: pytest.skip("openclaw CLI is not installed") - proc = subprocess.run( - [ - openclaw_bin, - "agent", - "exec", - "--help", - ], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - timeout=10, - ) + try: + proc = subprocess.run( + [ + openclaw_bin, + "infer", + "model", + "run", + "--help", + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=10, + ) + except subprocess.TimeoutExpired: + pytest.skip("installed openclaw CLI timed out while rendering infer model run help") output = proc.stdout + proc.stderr assert proc.returncode == 0 - assert "Usage: openclaw agent exec" in output - assert "--code-mode " in output - assert "direct" in output - assert "--message-file " in output + assert "Usage: openclaw infer model run" in output + assert "--local" in output + assert "--prompt " in output assert "--json" in output - assert "--timeout " in output + assert "--model " in output assert "--thinking " in output def test_classify_notification_with_llm_reports_timeout_without_prompt(monkeypatch): notif = notification("<1@github.com>", "@pilipilisbot review this") - assert_openclaw_agent_exec_supported.cache_clear() + assert_openclaw_model_run_supported.cache_clear() def fake_run(cmd, **kwargs): if cmd[-1] == "--help": return subprocess.CompletedProcess( cmd, 0, - "Usage: openclaw agent exec\n" - "--code-mode direct | auto | code\n" - "--message-file \n--json\n--timeout \n" + "Usage: openclaw infer model run\n" + "--local\n--prompt \n--json\n--model \n" "--thinking \n", "", ) @@ -372,7 +375,7 @@ def fake_run(cmd, **kwargs): def test_classify_notification_with_llm_retries_once(monkeypatch): notif = notification("<1@github.com>", "@pilipilisbot review this") calls = 0 - assert_openclaw_agent_exec_supported.cache_clear() + assert_openclaw_model_run_supported.cache_clear() def fake_run(cmd, **kwargs): nonlocal calls @@ -380,9 +383,8 @@ def fake_run(cmd, **kwargs): return subprocess.CompletedProcess( cmd, 0, - "Usage: openclaw agent exec\n" - "--code-mode direct | auto | code\n" - "--message-file \n--json\n--timeout \n" + "Usage: openclaw infer model run\n" + "--local\n--prompt \n--json\n--model \n" "--thinking \n", "", ) From f1a88cd6df10a6a96c8de1a820bde5113b97a986 Mon Sep 17 00:00:00 2001 From: Pilipilis Date: Fri, 9 Oct 2026 10:08:05 +0000 Subject: [PATCH 5/5] fix: parse raw model run outputs Read the OpenClaw raw inference envelope outputs text field before normalizing classifier results, and cover the contract with a regression test.\n\nCo-authored-by: giscebot <286264155+giscebot@users.noreply.github.com> --- src/github_agent_bridge/intent_classifier.py | 12 ++++ tests/test_intent_classifier.py | 63 +++++++++++++++----- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/src/github_agent_bridge/intent_classifier.py b/src/github_agent_bridge/intent_classifier.py index d4e4a0eb..a1b80dfd 100644 --- a/src/github_agent_bridge/intent_classifier.py +++ b/src/github_agent_bridge/intent_classifier.py @@ -94,6 +94,12 @@ def _text_from_model_run_json(raw: str) -> str: data.get("message"), data.get("response"), ] + outputs = data.get("outputs") + if isinstance(outputs, list): + for output in outputs: + if isinstance(output, dict): + candidates.append(output.get("text")) + candidates.append(output.get("content")) result = data.get("result") if isinstance(result, dict): candidates.extend( @@ -106,6 +112,12 @@ def _text_from_model_run_json(raw: str) -> str: result.get("response"), ] ) + outputs = result.get("outputs") + if isinstance(outputs, list): + for output in outputs: + if isinstance(output, dict): + candidates.append(output.get("text")) + candidates.append(output.get("content")) payloads = result.get("payloads") if isinstance(payloads, list): for payload in payloads: diff --git a/tests/test_intent_classifier.py b/tests/test_intent_classifier.py index 75e81f31..f3d5231a 100644 --- a/tests/test_intent_classifier.py +++ b/tests/test_intent_classifier.py @@ -5,6 +5,7 @@ from github_agent_bridge.intent_classifier import ( ParserResult, + _text_from_model_run_json, assert_openclaw_model_run_supported, build_intent_prompt, classify_notification_with_llm, @@ -215,6 +216,37 @@ def test_normalize_result_requires_write_permission_for_work_allowed(): assert result.work_intent == "review_only" +def test_model_run_json_reads_raw_inference_outputs_text(): + text = _text_from_model_run_json( + json.dumps( + { + "ok": True, + "outputs": [ + { + "text": json.dumps( + { + "addressed_to_agent": True, + "action": "reply_comment", + "work_intent": "work_allowed", + "write_permission": "state_change_allowed", + "confidence": 0.92, + "reason": "User asked for implementation.", + } + ), + "mediaUrl": None, + } + ], + } + ) + ) + + result = normalize_result(json.loads(text), 0.75) + + assert result.applied is True + assert result.action == "reply_comment" + assert result.work_intent == "work_allowed" + + def test_classify_notification_with_llm_uses_toolless_model_run(monkeypatch): calls = [] assert_openclaw_model_run_supported.cache_clear() @@ -235,22 +267,21 @@ def fake_run(cmd, **kwargs): 0, json.dumps( { - "result": { - "payloads": [ - { - "text": json.dumps( - { - "addressed_to_agent": True, - "action": "reply_comment", - "work_intent": "work_allowed", - "write_permission": "state_change_allowed", - "confidence": 0.92, - "reason": "User asked for implementation.", - } - ) - } - ] - } + "ok": True, + "outputs": [ + { + "text": json.dumps( + { + "addressed_to_agent": True, + "action": "reply_comment", + "work_intent": "work_allowed", + "write_permission": "state_change_allowed", + "confidence": 0.92, + "reason": "User asked for implementation.", + } + ) + } + ], } ), "",