From 2dceb6ff8ff73576a837f0970486e0436b07aa75 Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Mon, 2 Nov 2026 10:58:23 -0500 Subject: [PATCH 1/6] feat(firefox): share CI build setup across agents --- .../hackbot_agents/build_repair/__main__.py | 3 +- .../hackbot_agents/build_repair/agent.py | 38 +----- .../hackbot_agents/build_repair/prompts.py | 9 +- .../hackbot_agents/build_repair/resolve.py | 25 ---- agents/build-repair/tests/test_ci_build.py | 89 -------------- agents/build-repair/tests/test_reporting.py | 31 ++++- docs/hackbot/tools.md | 15 +++ .../agent_tools/firefox/__init__.py | 36 +++++- .../agent_tools/firefox/ci_build.py | 97 +++++++++++++++ .../firefox/tools/build_firefox.py | 3 + libs/agent-tools/tests/test_firefox_ci.py | 113 ++++++++++++++++++ 11 files changed, 298 insertions(+), 161 deletions(-) delete mode 100644 agents/build-repair/tests/test_ci_build.py create mode 100644 libs/agent-tools/agent_tools/firefox/ci_build.py create mode 100644 libs/agent-tools/tests/test_firefox_ci.py diff --git a/agents/build-repair/hackbot_agents/build_repair/__main__.py b/agents/build-repair/hackbot_agents/build_repair/__main__.py index 222ef4a10b1..067c6ae37f4 100644 --- a/agents/build-repair/hackbot_agents/build_repair/__main__.py +++ b/agents/build-repair/hackbot_agents/build_repair/__main__.py @@ -1,5 +1,6 @@ import logging +from agent_tools.firefox import linux_build_env from hackbot_runtime import HackbotContext, run_async from hackbot_runtime.actions.email import record_email from hackbot_runtime.actions.phabricator import PATCH_ACTION_TYPES @@ -8,7 +9,7 @@ from .agent import BuildRepairResult, run_build_repair from .config import CHECKOUT_DEPTH, NOTIFY_ONLY_WITH_PATCH from .notify import build_email, recipients, resolve_author_email -from .resolve import PushInfo, linux_build_env, resolve_push +from .resolve import PushInfo, resolve_push logger = logging.getLogger(__name__) diff --git a/agents/build-repair/hackbot_agents/build_repair/agent.py b/agents/build-repair/hackbot_agents/build_repair/agent.py index 7ac50bc2ea2..a783b4b08f2 100644 --- a/agents/build-repair/hackbot_agents/build_repair/agent.py +++ b/agents/build-repair/hackbot_agents/build_repair/agent.py @@ -14,7 +14,6 @@ import contextlib import json -import os import re import subprocess import sys @@ -137,37 +136,6 @@ def _build_options( ) -def _setup_ci_build(fx_ctx: FirefoxContext, env: dict[str, str]) -> None: - """Build as the failing CI task did: its toolchains and mozconfig.""" - fetches = [f"{f['task']}:{f['artifact']}" for f in json.loads(env["MOZ_FETCHES"])] - fetches_dir = fx_ctx.source_dir.parent / "fetches" - fetches_dir.mkdir(exist_ok=True) - # A reused workspace keeps the in-tree mozconfig, which mach refuses outside automation. - fx_ctx.mozconfig.unlink(missing_ok=True) - subprocess.run( - [str(fx_ctx.source_dir / "mach"), "--no-interactive", "artifact"] - + ["toolchain", "--from-task", *fetches], - cwd=fetches_dir, - check=True, - stdout=sys.stderr, - stderr=sys.stderr, - ) - # mach takes in-tree mozconfigs only in automation. - os.environ.update( - { - k: env[k] - for k in ("DISABLE_CLANG_PLUGIN", "FORCE_GCC", "PERFHERDER_EXTRA_OPTIONS") - if k in env - }, - MOZ_AUTOMATION="1", - MOZ_FETCHES_DIR=str(fetches_dir), - ) - fx_ctx.mozconfig.write_text( - f'. "$topsrcdir/{env["MOZCONFIG"]}"\n' - f"mk_add_options MOZ_OBJDIR={fx_ctx.objdir}\n" - ) - - def _write_mozconfig(fx_ctx: FirefoxContext) -> None: """Write a mozconfig mirroring the failing CI build, unless one exists. @@ -250,6 +218,8 @@ async def run_build_repair( scratch_out = scratch_dir / "out" scratch_out.mkdir(parents=True, exist_ok=True) + if ci_env is not None: + fx_ctx.configure_ci_build(ci_env) firefox_tools = [*firefox.TOOLS, *TRY_TOOLS] if run_try_push else firefox.TOOLS firefox_server = build_sdk_server("firefox", fx_ctx, firefox_tools) mcp_servers: dict[str, McpServerConfig] = { @@ -398,9 +368,7 @@ async def run_build_repair( # Stage 2: fix (lower effort, edits the source tree and verifies it # builds against a mozconfig that mirrors the failing CI config). - if ci_env: - _setup_ci_build(fx_ctx, ci_env) - else: + if not ci_env: _write_mozconfig(fx_ctx) fix_label = ( f"bug {resolved_bug_id}" if resolved_bug_id is not None else label diff --git a/agents/build-repair/hackbot_agents/build_repair/prompts.py b/agents/build-repair/hackbot_agents/build_repair/prompts.py index 65515f64275..921f89a5421 100644 --- a/agents/build-repair/hackbot_agents/build_repair/prompts.py +++ b/agents/build-repair/hackbot_agents/build_repair/prompts.py @@ -170,11 +170,12 @@ been read, which costs a turn. To see how a commit handled comparable files, run `git show -- ` rather than guessing a sibling's name. -Logs already fetched sit under {scratch_out}/logs. A mozconfig that mirrors the failing CI configuration is already set up. Verify the fix compiles with the build_firefox tool, passing +Logs already fetched sit under {scratch_out}/logs. When the failing task has a supported Linux CI configuration, the first build_firefox call fetches its toolchains and writes the matching mozconfig. Otherwise the default mozconfig is already set up. Verify the fix compiles with the build_firefox tool, passing the directory of the file you changed as `target` (e.g. 'docshell/base') for a -fast, focused build -- prefer this over a full tree build. If the build reports a -missing toolchain (e.g. rustc or clang), run the bootstrap_firefox tool once and -then build again. Verify via the build_firefox tool rather than a raw `./mach +fast, focused build -- prefer this over a full tree build. If using the CI +configuration, do not replace its toolchains with bootstrap; report a setup +failure. For the default configuration, bootstrap once if the build reports a +missing toolchain. Verify via the build_firefox tool rather than a raw `./mach build` so the build result is recorded. {try_push}{report}""" diff --git a/agents/build-repair/hackbot_agents/build_repair/resolve.py b/agents/build-repair/hackbot_agents/build_repair/resolve.py index 87e822a11f6..6d2e26276c3 100644 --- a/agents/build-repair/hackbot_agents/build_repair/resolve.py +++ b/agents/build-repair/hackbot_agents/build_repair/resolve.py @@ -12,9 +12,7 @@ derives everything it reports from a task id. """ -import json import logging -import platform import re from dataclasses import dataclass, field @@ -53,29 +51,6 @@ def _task(task_id: str) -> dict: return _get_json(_TC_TASK_URL.format(task_id=task_id)) -def linux_build_env(task_id: str) -> dict[str, str] | None: - """The env of a build task CI ran on Linux, or None for any other task. - - Its ``MOZCONFIG`` and ``MOZ_FETCHES`` say how to build exactly as CI did. - Fetches with a ``dest`` or without extraction are not supported, and CI's - toolchains only run on an x86-64 host. - """ - if platform.machine() != "x86_64": - return None - task = _task(task_id) - env = (task.get("payload") or {}).get("env") or {} - if not ( - task.get("workerType", "").startswith("b-linux") - and env.get("MOZCONFIG") - and env.get("MOZ_FETCHES") - ): - return None - fetches = json.loads(env["MOZ_FETCHES"]) - if any("dest" in f or not f.get("extract", True) for f in fetches): - return None - return env - - def _task_env(task: dict) -> dict: return (task.get("payload") or {}).get("env") or {} diff --git a/agents/build-repair/tests/test_ci_build.py b/agents/build-repair/tests/test_ci_build.py deleted file mode 100644 index 15381ea10a9..00000000000 --- a/agents/build-repair/tests/test_ci_build.py +++ /dev/null @@ -1,89 +0,0 @@ -import json -import os - -from agent_tools.firefox import FirefoxContext -from hackbot_agents.build_repair import agent, resolve - -FETCHES = [ - {"artifact": "public/build/clang.tar.zst", "extract": True, "task": "CLANG"}, - {"artifact": "public/build/sccache.tar.zst", "extract": True, "task": "SCCACHE"}, -] -ENV = { - "MOZCONFIG": "browser/config/mozconfigs/linux64/nightly", - "MOZ_FETCHES": json.dumps(FETCHES), - "PERFHERDER_EXTRA_OPTIONS": "x11", - "DISABLE_CLANG_PLUGIN": "1", -} - - -def _task(worker_type, env): - return {"workerType": worker_type, "payload": {"env": env}} - - -def test_linux_build_env_takes_linux_build_tasks(monkeypatch): - monkeypatch.setattr(resolve.platform, "machine", lambda: "x86_64") - monkeypatch.setattr(resolve, "_task", lambda _: _task("b-linux-docker-amd", ENV)) - assert resolve.linux_build_env("T") == ENV - - -def test_linux_build_env_needs_an_x86_64_host(monkeypatch): - monkeypatch.setattr(resolve.platform, "machine", lambda: "aarch64") - monkeypatch.setattr(resolve, "_task", lambda _: _task("b-linux-docker-amd", ENV)) - assert resolve.linux_build_env("T") is None - - -def test_linux_build_env_skips_other_tasks(monkeypatch): - monkeypatch.setattr(resolve.platform, "machine", lambda: "x86_64") - for task in ( - _task("b-osx-arm64", ENV), - _task("b-linux-docker-amd", {"MOZ_FETCHES": ENV["MOZ_FETCHES"]}), - _task( - "b-linux-docker-amd", - {**ENV, "MOZ_FETCHES": json.dumps([{**FETCHES[0], "dest": "sdk"}])}, - ), - ): - monkeypatch.setattr(resolve, "_task", lambda _, task=task: task) - assert resolve.linux_build_env("T") is None - - -def test_setup_ci_build_fetches_ci_toolchains_and_uses_ci_mozconfig( - tmp_path, monkeypatch -): - src = tmp_path / "firefox" - src.mkdir() - mach = src / "mach" - mach.write_text( - f'#!/bin/sh\n[ -e "{src}/.mozconfig" ] && exit 1\n' - f'echo "$(pwd -P) $*" > "{tmp_path}/mach.log"\n' - ) - mach.chmod(0o755) - fetches = tmp_path / "fetches" - for var in ( - "MOZ_AUTOMATION", - "MOZ_FETCHES_DIR", - "PERFHERDER_EXTRA_OPTIONS", - "DISABLE_CLANG_PLUGIN", - ): - monkeypatch.delenv(var, raising=False) - fx_ctx = FirefoxContext.from_source_repo(src, objdir=str(tmp_path / "obj-build")) - fx_ctx.mozconfig.write_text("left by an earlier run\n") - - agent._setup_ci_build(fx_ctx, ENV) - - assert (tmp_path / "mach.log").read_text().split() == [ - str(fetches.resolve()), - "--no-interactive", - "artifact", - "toolchain", - "--from-task", - "CLANG:public/build/clang.tar.zst", - "SCCACHE:public/build/sccache.tar.zst", - ] - assert os.environ["MOZ_AUTOMATION"] == "1" - assert os.environ["MOZ_FETCHES_DIR"] == str(fetches) - assert os.environ["PERFHERDER_EXTRA_OPTIONS"] == "x11" - assert os.environ["DISABLE_CLANG_PLUGIN"] == "1" - assert fx_ctx.mozconfig.read_text().splitlines() == [ - '. "$topsrcdir/browser/config/mozconfigs/linux64/nightly"', - f"mk_add_options MOZ_OBJDIR={tmp_path / 'obj-build'}", - ] diff --git a/agents/build-repair/tests/test_reporting.py b/agents/build-repair/tests/test_reporting.py index 064e70c1a06..6812077d028 100644 --- a/agents/build-repair/tests/test_reporting.py +++ b/agents/build-repair/tests/test_reporting.py @@ -36,6 +36,8 @@ def _run( push_bug=None, blame=None, checkout=None, + ci_env=None, + fx_ctx=None, ): """Run the agent with both sessions faked, returning their (options, prompt). @@ -60,19 +62,21 @@ async def fake_session(reporter, options, prompt, captured, tracked): monkeypatch.setattr(agent, "_run_session", fake_session) monkeypatch.setattr(agent, "build_sdk_server", lambda *a, **k: {"type": "sdk"}) + fx_ctx = fx_ctx or SimpleNamespace( + mozconfig=tmp_path / ".mozconfig", objdir=tmp_path / "objdir" + ) result = asyncio.run( agent.run_build_repair( bugzilla_mcp_server={"type": "http", "url": "http://broker/mcp"}, source_repo=tmp_path, - fx_ctx=SimpleNamespace( - mozconfig=tmp_path / ".mozconfig", objdir=tmp_path / "objdir" - ), + fx_ctx=fx_ctx, bug_id=bug_id, commit_bugs={FAILURE_COMMIT: push_bug} if push_bug else {}, git_commits=[FAILURE_COMMIT], project="autoland", hg_revision="abc123", failure_tasks={"build-linux": "taskid"}, + ci_env=ci_env, actions_recorder=actions_recorder, checkout=checkout, ) @@ -80,6 +84,27 @@ async def fake_session(reporter, options, prompt, captured, tracked): return result, sessions[0], sessions[1] if len(sessions) > 1 else None +def test_ci_build_env_is_attached_to_firefox_context(tmp_path, monkeypatch): + env = {"MOZCONFIG": "browser/config/mozconfigs/linux64/nightly"} + configured = [] + fx_ctx = SimpleNamespace( + mozconfig=tmp_path / ".mozconfig", + objdir=tmp_path / "objdir", + configure_ci_build=configured.append, + ) + + _run( + tmp_path, + monkeypatch, + bug_id=1, + actions_recorder=None, + ci_env=env, + fx_ctx=fx_ctx, + ) + + assert configured == [env] + + # --- the fix stage works on the blamed commit itself ----------------------- # diff --git a/docs/hackbot/tools.md b/docs/hackbot/tools.md index 411e71fb3ad..a0e87e1c6e9 100644 --- a/docs/hackbot/tools.md +++ b/docs/hackbot/tools.md @@ -128,6 +128,21 @@ locally against the checkout. full tree, much less incrementally. An agent that only needs to confirm a localized fix compiles should pass `target` to build one directory instead of the whole tree. +For a CI reproduction, an agent with a failed Taskcluster task can configure the Firefox +context before starting the tool server: + +```python +from agent_tools.firefox import linux_build_env + +ctx.firefox.configure_ci_build(linux_build_env(task_id)) +``` + +On supported x86-64 Linux tasks, `build_firefox` then fetches the task's toolchains and +writes its mozconfig on the first call. The context reuses that setup for later builds in +the same run; a new context removes a stale in-tree mozconfig before fetching, so local +runs can reuse the workspace volume. Unsupported tasks and non-x86-64 hosts keep the +agent's normal build configuration. + ## Adding a tool 1. Write the handler in the right domain module, `@tool`-decorated, `ctx` first, returning diff --git a/libs/agent-tools/agent_tools/firefox/__init__.py b/libs/agent-tools/agent_tools/firefox/__init__.py index 9a16127c4ed..f6544041e49 100644 --- a/libs/agent-tools/agent_tools/firefox/__init__.py +++ b/libs/agent-tools/agent_tools/firefox/__init__.py @@ -5,7 +5,7 @@ first parameter and returns plain data. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Annotated, Literal @@ -13,6 +13,8 @@ from agent_tools.registry import tool, tools_in +from .ci_build import linux_build_env as linux_build_env +from .ci_build import prepare_ci_build from .tools import bootstrap_firefox as _bootstrap_firefox from .tools import build_firefox as _build_firefox from .tools import evaluate_testcase as _evaluate_testcase @@ -35,6 +37,14 @@ class FirefoxContext: js_binary: Path # What bootstrap_firefox last set up, which build_firefox then builds for. application: str = "browser" + # Optional task env to mirror when build_firefox is called. + ci_env: dict[str, str] | None = field(default=None, repr=False) + _ci_build_env: dict[str, str] | None = field(default=None, init=False, repr=False) + + def configure_ci_build(self, env: dict[str, str] | None) -> None: + """Use a supported CI task's env for the next Firefox build.""" + self.ci_env = env + self._ci_build_env = None @property def android_mozconfig(self) -> Path: @@ -157,8 +167,10 @@ async def build_firefox( For Android once ``bootstrap_firefox`` has been called for it, with the mozconfig it wrote for that; for desktop Firefox otherwise. - Slow on a full tree build (tens of minutes cold, faster incremental); pass a - `target` directory to build just the part you changed. Returns JSON: success + When configured with a supported CI task env, the first call fetches its + toolchains and uses its mozconfig. Slow on a full tree build (tens of minutes + cold, faster incremental); pass a `target` directory to build just the part + you changed. Returns JSON: success (bool), build_dir (str), message (str), stdout/stderr. Only call this if you've changed source or the binary is missing — check if the binary exists first. @@ -166,7 +178,23 @@ async def build_firefox( firefox_dir_p = Path(firefox_dir) if firefox_dir else ctx.source_dir mozconfig, objdir = ctx.build_config() mozconfig_p = Path(mozconfig_path) if mozconfig_path else mozconfig - return await _build_firefox(firefox_dir_p, mozconfig_p, objdir, target=target) + if ctx.ci_env is not None and ctx._ci_build_env is None: + try: + ctx._ci_build_env = await prepare_ci_build( + firefox_dir_p, mozconfig_p, objdir, ctx.ci_env + ) + except Exception as e: + return { + "success": False, + "message": f"Error preparing the CI Firefox build: {type(e).__name__}: {e!s}", + } + return await _build_firefox( + firefox_dir_p, + mozconfig_p, + objdir, + target=target, + env_overrides=ctx._ci_build_env, + ) @tool diff --git a/libs/agent-tools/agent_tools/firefox/ci_build.py b/libs/agent-tools/agent_tools/firefox/ci_build.py new file mode 100644 index 00000000000..d8164b44c38 --- /dev/null +++ b/libs/agent-tools/agent_tools/firefox/ci_build.py @@ -0,0 +1,97 @@ +"""Build Firefox with the toolchains and mozconfig from a Linux CI task.""" + +import asyncio +import json +import os +import platform +from pathlib import Path +from urllib.request import Request, urlopen + +_TASK_URL = "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{task_id}" +_HEADERS = {"User-Agent": "hackbot-agent-tools/1.0"} +_TIMEOUT = 30 + + +def _task(task_id: str) -> dict: + request = Request(_TASK_URL.format(task_id=task_id), headers=_HEADERS) + with urlopen(request, timeout=_TIMEOUT) as response: + return json.load(response) + + +def linux_build_env(task_id: str) -> dict[str, str] | None: + """Return a supported Linux build task's env, or ``None`` for other tasks. + + CI's toolchains only run on an x86-64 host. Fetches with a custom ``dest`` + or without extraction are not supported by the Firefox build tool. + """ + if platform.machine() != "x86_64": + return None + + task = _task(task_id) + env = (task.get("payload") or {}).get("env") or {} + if not ( + task.get("workerType", "").startswith("b-linux") + and env.get("MOZCONFIG") + and env.get("MOZ_FETCHES") + ): + return None + + fetches = json.loads(env["MOZ_FETCHES"]) + if any("dest" in fetch or not fetch.get("extract", True) for fetch in fetches): + return None + return env + + +async def prepare_ci_build( + firefox_dir: Path, + mozconfig_path: Path, + objdir: Path, + task_env: dict[str, str], +) -> dict[str, str]: + """Fetch the CI toolchains and return env overrides for subsequent builds.""" + if platform.machine() != "x86_64": + raise RuntimeError("CI Firefox toolchains require an x86-64 host") + + firefox_dir = firefox_dir.resolve() + mozconfig_path = mozconfig_path.resolve() + objdir = objdir.resolve() + fetches = [ + f"{fetch['task']}:{fetch['artifact']}" + for fetch in json.loads(task_env["MOZ_FETCHES"]) + ] + fetches_dir = firefox_dir.parent / "fetches" + fetches_dir.mkdir(exist_ok=True) + + # A reused workspace may contain a mozconfig from the last run. mach refuses + # to fetch toolchains with that in-tree config unless automation is enabled. + mozconfig_path.unlink(missing_ok=True) + process = await asyncio.create_subprocess_exec( + str(firefox_dir / "mach"), + "--no-interactive", + "artifact", + "toolchain", + "--from-task", + *fetches, + cwd=fetches_dir, + env=os.environ.copy(), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await process.communicate() + if process.returncode != 0: + detail = (stderr or stdout).decode("utf-8", errors="replace").strip() + raise RuntimeError( + f"mach artifact toolchain failed with exit code {process.returncode}" + + (f": {detail}" if detail else "") + ) + + overrides = { + key: task_env[key] + for key in ("DISABLE_CLANG_PLUGIN", "FORCE_GCC", "PERFHERDER_EXTRA_OPTIONS") + if key in task_env + } + overrides.update(MOZ_AUTOMATION="1", MOZ_FETCHES_DIR=str(fetches_dir)) + mozconfig_path.write_text( + f'. "$topsrcdir/{task_env["MOZCONFIG"]}"\nmk_add_options MOZ_OBJDIR={objdir}\n' + ) + return overrides diff --git a/libs/agent-tools/agent_tools/firefox/tools/build_firefox.py b/libs/agent-tools/agent_tools/firefox/tools/build_firefox.py index 2c1a456b08c..ab664ba7397 100644 --- a/libs/agent-tools/agent_tools/firefox/tools/build_firefox.py +++ b/libs/agent-tools/agent_tools/firefox/tools/build_firefox.py @@ -11,6 +11,7 @@ async def build_firefox( mozconfig_path: Path, objdir: Path, target: str | None = None, + env_overrides: dict[str, str] | None = None, ) -> dict[str, Any]: """Build Firefox using a specified mozconfig. @@ -23,6 +24,7 @@ async def build_firefox( target: Optional build target (e.g. a directory like ``docshell/base``). When set, only that target is built -- far faster than a full tree build and enough to verify a localized fix compiles. + env_overrides: Environment required by a task-specific CI build. Returns: Dict with build result information (success, build_dir, message, @@ -42,6 +44,7 @@ async def build_firefox( } env = os.environ.copy() + env.update(env_overrides or {}) env["MOZCONFIG"] = str(mozconfig_path.resolve()) env["CLAUDECODE"] = "1" diff --git a/libs/agent-tools/tests/test_firefox_ci.py b/libs/agent-tools/tests/test_firefox_ci.py new file mode 100644 index 00000000000..37280b426c0 --- /dev/null +++ b/libs/agent-tools/tests/test_firefox_ci.py @@ -0,0 +1,113 @@ +import json +import os + +from agent_tools import firefox +from agent_tools.firefox import ci_build + +FETCHES = [ + {"artifact": "public/build/clang.tar.zst", "extract": True, "task": "CLANG"}, + {"artifact": "public/build/sccache.tar.zst", "extract": True, "task": "SCCACHE"}, +] +ENV = { + "MOZCONFIG": "browser/config/mozconfigs/linux64/nightly", + "MOZ_FETCHES": json.dumps(FETCHES), + "PERFHERDER_EXTRA_OPTIONS": "x11", + "DISABLE_CLANG_PLUGIN": "1", +} + + +def _task(worker_type, env): + return {"workerType": worker_type, "payload": {"env": env}} + + +def test_linux_build_env_takes_linux_build_tasks(monkeypatch): + monkeypatch.setattr(ci_build.platform, "machine", lambda: "x86_64") + monkeypatch.setattr(ci_build, "_task", lambda _: _task("b-linux-docker-amd", ENV)) + assert ci_build.linux_build_env("T") == ENV + + +def test_linux_build_env_needs_an_x86_64_host(monkeypatch): + monkeypatch.setattr(ci_build.platform, "machine", lambda: "aarch64") + monkeypatch.setattr(ci_build, "_task", lambda _: _task("b-linux-docker-amd", ENV)) + assert ci_build.linux_build_env("T") is None + + +def test_linux_build_env_skips_unsupported_tasks(monkeypatch): + monkeypatch.setattr(ci_build.platform, "machine", lambda: "x86_64") + tasks = ( + _task("b-osx-arm64", ENV), + _task("b-linux-docker-amd", {"MOZ_FETCHES": ENV["MOZ_FETCHES"]}), + _task( + "b-linux-docker-amd", + {**ENV, "MOZ_FETCHES": json.dumps([{**FETCHES[0], "dest": "sdk"}])}, + ), + _task( + "b-linux-docker-amd", + { + **ENV, + "MOZ_FETCHES": json.dumps([{**FETCHES[0], "extract": False}]), + }, + ), + ) + for task in tasks: + monkeypatch.setattr(ci_build, "_task", lambda _, task=task: task) + assert ci_build.linux_build_env("T") is None + + +async def test_build_firefox_prepares_ci_build_once_per_context_and_resets_stale_mozconfig( + tmp_path, monkeypatch +): + monkeypatch.setattr(ci_build.platform, "machine", lambda: "x86_64") + monkeypatch.delenv("MOZ_AUTOMATION", raising=False) + monkeypatch.delenv("MOZ_FETCHES_DIR", raising=False) + monkeypatch.delenv("PERFHERDER_EXTRA_OPTIONS", raising=False) + monkeypatch.delenv("DISABLE_CLANG_PLUGIN", raising=False) + + src = tmp_path / "firefox" + src.mkdir() + objdir = src / "objdir" + (objdir / "build" / "clang-plugin").mkdir(parents=True) + mach_log = tmp_path / "mach.log" + mach = src / "mach" + mach.write_text( + "#!/bin/sh\n" + f'if [ "$1" = "--no-interactive" ]; then ' + f'[ ! -e "{src}/.mozconfig" ] || exit 41; fi\n' + f'echo "$*" >> "{mach_log}"\n' + ) + mach.chmod(0o755) + (src / ".mozconfig").write_text("left by the previous run\n") + + first = firefox.FirefoxContext.from_source_repo(src, objdir=str(objdir)) + first.configure_ci_build(ENV) + assert (await firefox.build_firefox(first, target="image"))["success"] + assert (await firefox.build_firefox(first, target="image"))["success"] + + # A new runtime context models the next local run reusing the same volume. + second = firefox.FirefoxContext.from_source_repo(src, objdir=str(objdir)) + second.configure_ci_build(ENV) + assert (await firefox.build_firefox(second, target="image"))["success"] + + commands = mach_log.read_text().splitlines() + assert commands == [ + "--no-interactive artifact toolchain --from-task " + "CLANG:public/build/clang.tar.zst SCCACHE:public/build/sccache.tar.zst", + "build --quiet pre-export export", + "build --quiet --allow-subdirectory-build build/clang-plugin image", + "build --quiet pre-export export", + "build --quiet --allow-subdirectory-build build/clang-plugin image", + "--no-interactive artifact toolchain --from-task " + "CLANG:public/build/clang.tar.zst SCCACHE:public/build/sccache.tar.zst", + "build --quiet pre-export export", + "build --quiet --allow-subdirectory-build build/clang-plugin image", + ] + assert "MOZ_AUTOMATION" not in os.environ + assert "MOZ_FETCHES_DIR" not in os.environ + assert "PERFHERDER_EXTRA_OPTIONS" not in os.environ + assert "DISABLE_CLANG_PLUGIN" not in os.environ + assert first._ci_build_env == { + "PERFHERDER_EXTRA_OPTIONS": "x11", + "DISABLE_CLANG_PLUGIN": "1", + "MOZ_AUTOMATION": "1", + "MOZ_FETCHES_DIR": str(tmp_path / "fetches"), + } From d964941b5acdf4467d903347be507cf718d5515c Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Mon, 2 Nov 2026 11:03:40 -0500 Subject: [PATCH 2/6] refactor(firefox): make CI build setup explicit --- .../hackbot_agents/build_repair/agent.py | 6 +-- .../hackbot_agents/build_repair/prompts.py | 3 +- agents/build-repair/tests/test_reporting.py | 22 +++++----- docs/hackbot/tools.md | 12 +++--- .../agent_tools/firefox/__init__.py | 29 +++++--------- .../agent_tools/firefox/ci_build.py | 2 - libs/agent-tools/tests/test_firefox_ci.py | 40 ++++++++++++++----- 7 files changed, 60 insertions(+), 54 deletions(-) diff --git a/agents/build-repair/hackbot_agents/build_repair/agent.py b/agents/build-repair/hackbot_agents/build_repair/agent.py index a783b4b08f2..72f27cc9234 100644 --- a/agents/build-repair/hackbot_agents/build_repair/agent.py +++ b/agents/build-repair/hackbot_agents/build_repair/agent.py @@ -218,8 +218,6 @@ async def run_build_repair( scratch_out = scratch_dir / "out" scratch_out.mkdir(parents=True, exist_ok=True) - if ci_env is not None: - fx_ctx.configure_ci_build(ci_env) firefox_tools = [*firefox.TOOLS, *TRY_TOOLS] if run_try_push else firefox.TOOLS firefox_server = build_sdk_server("firefox", fx_ctx, firefox_tools) mcp_servers: dict[str, McpServerConfig] = { @@ -368,7 +366,9 @@ async def run_build_repair( # Stage 2: fix (lower effort, edits the source tree and verifies it # builds against a mozconfig that mirrors the failing CI config). - if not ci_env: + if ci_env: + await fx_ctx.configure_ci_build(ci_env) + else: _write_mozconfig(fx_ctx) fix_label = ( f"bug {resolved_bug_id}" if resolved_bug_id is not None else label diff --git a/agents/build-repair/hackbot_agents/build_repair/prompts.py b/agents/build-repair/hackbot_agents/build_repair/prompts.py index 921f89a5421..7ad098fda6e 100644 --- a/agents/build-repair/hackbot_agents/build_repair/prompts.py +++ b/agents/build-repair/hackbot_agents/build_repair/prompts.py @@ -170,7 +170,8 @@ been read, which costs a turn. To see how a commit handled comparable files, run `git show -- ` rather than guessing a sibling's name. -Logs already fetched sit under {scratch_out}/logs. When the failing task has a supported Linux CI configuration, the first build_firefox call fetches its toolchains and writes the matching mozconfig. Otherwise the default mozconfig is already set up. Verify the fix compiles with the build_firefox tool, passing +Logs already fetched sit under {scratch_out}/logs. The mozconfig and any supported +CI toolchains are already set up. Verify the fix compiles with build_firefox, passing the directory of the file you changed as `target` (e.g. 'docshell/base') for a fast, focused build -- prefer this over a full tree build. If using the CI configuration, do not replace its toolchains with bootstrap; report a setup diff --git a/agents/build-repair/tests/test_reporting.py b/agents/build-repair/tests/test_reporting.py index 6812077d028..6fe963dc0d3 100644 --- a/agents/build-repair/tests/test_reporting.py +++ b/agents/build-repair/tests/test_reporting.py @@ -37,7 +37,6 @@ def _run( blame=None, checkout=None, ci_env=None, - fx_ctx=None, ): """Run the agent with both sessions faked, returning their (options, prompt). @@ -62,14 +61,11 @@ async def fake_session(reporter, options, prompt, captured, tracked): monkeypatch.setattr(agent, "_run_session", fake_session) monkeypatch.setattr(agent, "build_sdk_server", lambda *a, **k: {"type": "sdk"}) - fx_ctx = fx_ctx or SimpleNamespace( - mozconfig=tmp_path / ".mozconfig", objdir=tmp_path / "objdir" - ) result = asyncio.run( agent.run_build_repair( bugzilla_mcp_server={"type": "http", "url": "http://broker/mcp"}, source_repo=tmp_path, - fx_ctx=fx_ctx, + fx_ctx=agent.FirefoxContext.from_source_repo(tmp_path, objdir="objdir"), bug_id=bug_id, commit_bugs={FAILURE_COMMIT: push_bug} if push_bug else {}, git_commits=[FAILURE_COMMIT], @@ -84,25 +80,25 @@ async def fake_session(reporter, options, prompt, captured, tracked): return result, sessions[0], sessions[1] if len(sessions) > 1 else None -def test_ci_build_env_is_attached_to_firefox_context(tmp_path, monkeypatch): +@pytest.mark.parametrize("blame", [None, ""]) +def test_ci_build_is_configured_only_for_the_fix_stage(tmp_path, monkeypatch, blame): env = {"MOZCONFIG": "browser/config/mozconfigs/linux64/nightly"} configured = [] - fx_ctx = SimpleNamespace( - mozconfig=tmp_path / ".mozconfig", - objdir=tmp_path / "objdir", - configure_ci_build=configured.append, - ) + async def configure(self, env): + configured.append(env) + + monkeypatch.setattr(agent.FirefoxContext, "configure_ci_build", configure) _run( tmp_path, monkeypatch, bug_id=1, actions_recorder=None, ci_env=env, - fx_ctx=fx_ctx, + blame=blame, ) - assert configured == [env] + assert configured == ([env] if blame is None else []) # --- the fix stage works on the blamed commit itself ----------------------- # diff --git a/docs/hackbot/tools.md b/docs/hackbot/tools.md index a0e87e1c6e9..ba5aec70329 100644 --- a/docs/hackbot/tools.md +++ b/docs/hackbot/tools.md @@ -134,14 +134,14 @@ context before starting the tool server: ```python from agent_tools.firefox import linux_build_env -ctx.firefox.configure_ci_build(linux_build_env(task_id)) +await ctx.firefox.configure_ci_build(linux_build_env(task_id)) ``` -On supported x86-64 Linux tasks, `build_firefox` then fetches the task's toolchains and -writes its mozconfig on the first call. The context reuses that setup for later builds in -the same run; a new context removes a stale in-tree mozconfig before fetching, so local -runs can reuse the workspace volume. Unsupported tasks and non-x86-64 hosts keep the -agent's normal build configuration. +This fetches the task's toolchains and writes its mozconfig for supported Linux tasks +on x86-64 hosts. Call it once before building; later `build_firefox` calls reuse the +setup. It removes a stale in-tree mozconfig before fetching, so local runs can reuse +the workspace volume. CI environment overrides apply only to build subprocesses. +Unsupported tasks and non-x86-64 hosts keep the agent's normal build configuration. ## Adding a tool diff --git a/libs/agent-tools/agent_tools/firefox/__init__.py b/libs/agent-tools/agent_tools/firefox/__init__.py index f6544041e49..9853063da3d 100644 --- a/libs/agent-tools/agent_tools/firefox/__init__.py +++ b/libs/agent-tools/agent_tools/firefox/__init__.py @@ -37,14 +37,15 @@ class FirefoxContext: js_binary: Path # What bootstrap_firefox last set up, which build_firefox then builds for. application: str = "browser" - # Optional task env to mirror when build_firefox is called. - ci_env: dict[str, str] | None = field(default=None, repr=False) _ci_build_env: dict[str, str] | None = field(default=None, init=False, repr=False) - def configure_ci_build(self, env: dict[str, str] | None) -> None: - """Use a supported CI task's env for the next Firefox build.""" - self.ci_env = env - self._ci_build_env = None + async def configure_ci_build(self, env: dict[str, str] | None) -> None: + """Fetch a CI task's toolchains and use its mozconfig, or clear overrides.""" + self._ci_build_env = ( + await prepare_ci_build(self.source_dir, self.mozconfig, self.objdir, env) + if env is not None + else None + ) @property def android_mozconfig(self) -> Path: @@ -167,10 +168,8 @@ async def build_firefox( For Android once ``bootstrap_firefox`` has been called for it, with the mozconfig it wrote for that; for desktop Firefox otherwise. - When configured with a supported CI task env, the first call fetches its - toolchains and uses its mozconfig. Slow on a full tree build (tens of minutes - cold, faster incremental); pass a `target` directory to build just the part - you changed. Returns JSON: success + Slow on a full tree build (tens of minutes cold, faster incremental); pass a + `target` directory to build just the part you changed. Returns JSON: success (bool), build_dir (str), message (str), stdout/stderr. Only call this if you've changed source or the binary is missing — check if the binary exists first. @@ -178,16 +177,6 @@ async def build_firefox( firefox_dir_p = Path(firefox_dir) if firefox_dir else ctx.source_dir mozconfig, objdir = ctx.build_config() mozconfig_p = Path(mozconfig_path) if mozconfig_path else mozconfig - if ctx.ci_env is not None and ctx._ci_build_env is None: - try: - ctx._ci_build_env = await prepare_ci_build( - firefox_dir_p, mozconfig_p, objdir, ctx.ci_env - ) - except Exception as e: - return { - "success": False, - "message": f"Error preparing the CI Firefox build: {type(e).__name__}: {e!s}", - } return await _build_firefox( firefox_dir_p, mozconfig_p, diff --git a/libs/agent-tools/agent_tools/firefox/ci_build.py b/libs/agent-tools/agent_tools/firefox/ci_build.py index d8164b44c38..636fa2e0ba0 100644 --- a/libs/agent-tools/agent_tools/firefox/ci_build.py +++ b/libs/agent-tools/agent_tools/firefox/ci_build.py @@ -2,7 +2,6 @@ import asyncio import json -import os import platform from pathlib import Path from urllib.request import Request, urlopen @@ -73,7 +72,6 @@ async def prepare_ci_build( "--from-task", *fetches, cwd=fetches_dir, - env=os.environ.copy(), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) diff --git a/libs/agent-tools/tests/test_firefox_ci.py b/libs/agent-tools/tests/test_firefox_ci.py index 37280b426c0..17afd294d7d 100644 --- a/libs/agent-tools/tests/test_firefox_ci.py +++ b/libs/agent-tools/tests/test_firefox_ci.py @@ -1,6 +1,7 @@ import json import os +import pytest from agent_tools import firefox from agent_tools.firefox import ci_build @@ -54,7 +55,7 @@ def test_linux_build_env_skips_unsupported_tasks(monkeypatch): assert ci_build.linux_build_env("T") is None -async def test_build_firefox_prepares_ci_build_once_per_context_and_resets_stale_mozconfig( +async def test_ci_build_setup_supports_reused_workspaces_without_changing_process_env( tmp_path, monkeypatch ): monkeypatch.setattr(ci_build.platform, "machine", lambda: "x86_64") @@ -73,19 +74,25 @@ async def test_build_firefox_prepares_ci_build_once_per_context_and_resets_stale "#!/bin/sh\n" f'if [ "$1" = "--no-interactive" ]; then ' f'[ ! -e "{src}/.mozconfig" ] || exit 41; fi\n' + 'if [ "$1" = build ]; then\n' + ' [ "$MOZ_AUTOMATION" = 1 ] || exit 42\n' + f' [ "$MOZ_FETCHES_DIR" = "{tmp_path}/fetches" ] || exit 43\n' + ' [ "$DISABLE_CLANG_PLUGIN" = 1 ] || exit 44\n' + ' [ "$PERFHERDER_EXTRA_OPTIONS" = x11 ] || exit 45\n' + "fi\n" f'echo "$*" >> "{mach_log}"\n' ) mach.chmod(0o755) (src / ".mozconfig").write_text("left by the previous run\n") first = firefox.FirefoxContext.from_source_repo(src, objdir=str(objdir)) - first.configure_ci_build(ENV) + await first.configure_ci_build(ENV) assert (await firefox.build_firefox(first, target="image"))["success"] assert (await firefox.build_firefox(first, target="image"))["success"] # A new runtime context models the next local run reusing the same volume. second = firefox.FirefoxContext.from_source_repo(src, objdir=str(objdir)) - second.configure_ci_build(ENV) + await second.configure_ci_build(ENV) assert (await firefox.build_firefox(second, target="image"))["success"] commands = mach_log.read_text().splitlines() @@ -105,9 +112,24 @@ async def test_build_firefox_prepares_ci_build_once_per_context_and_resets_stale assert "MOZ_FETCHES_DIR" not in os.environ assert "PERFHERDER_EXTRA_OPTIONS" not in os.environ assert "DISABLE_CLANG_PLUGIN" not in os.environ - assert first._ci_build_env == { - "PERFHERDER_EXTRA_OPTIONS": "x11", - "DISABLE_CLANG_PLUGIN": "1", - "MOZ_AUTOMATION": "1", - "MOZ_FETCHES_DIR": str(tmp_path / "fetches"), - } + assert first.mozconfig.read_text().splitlines() == [ + '. "$topsrcdir/browser/config/mozconfigs/linux64/nightly"', + f"mk_add_options MOZ_OBJDIR={objdir}", + ] + await first.configure_ci_build(None) + assert first._ci_build_env is None + + +async def test_failed_ci_setup_does_not_write_a_build_mozconfig(tmp_path, monkeypatch): + monkeypatch.setattr(ci_build.platform, "machine", lambda: "x86_64") + mach = tmp_path / "mach" + mach.write_text("#!/bin/sh\necho toolchain-download-failed >&2\nexit 1\n") + mach.chmod(0o755) + ctx = firefox.FirefoxContext.from_source_repo(tmp_path) + ctx.mozconfig.write_text("left by the previous run\n") + + with pytest.raises(RuntimeError, match="toolchain-download-failed"): + await ctx.configure_ci_build(ENV) + + assert not ctx.mozconfig.exists() + assert ctx._ci_build_env is None From 004313af9d14d0cb61325f333b2aaf21778c59db Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Tue, 3 Nov 2026 11:37:34 -0500 Subject: [PATCH 3/6] feat(firefox): prepare native builds on demand --- agents/autowebcompat-intervention/Dockerfile | 4 +- agents/autowebcompat-intervention/compose.yml | 2 + agents/bug-fix/Dockerfile | 4 +- agents/bug-fix/compose.yml | 6 + .../bug-fix/hackbot_agents/bug_fix/agent.py | 1 + .../hackbot_agents/bug_fix/prompts/system.md | 4 +- agents/bug-fix/tests/test_build_setup.py | 38 +++++ agents/build-repair/Dockerfile | 4 +- agents/build-repair/compose.yml | 6 + docs/hackbot/tools.md | 44 ++++- .../agent_tools/firefox/__init__.py | 74 ++++++--- .../firefox/tools/build_firefox.py | 12 ++ libs/agent-tools/tests/test_firefox.py | 37 +++++ libs/agent-tools/tests/test_firefox_ci.py | 1 + .../tests/test_firefox_preparation.py | 157 ++++++++++++++++++ 15 files changed, 361 insertions(+), 33 deletions(-) create mode 100644 agents/bug-fix/tests/test_build_setup.py create mode 100644 libs/agent-tools/tests/test_firefox_preparation.py diff --git a/agents/autowebcompat-intervention/Dockerfile b/agents/autowebcompat-intervention/Dockerfile index 16219bedcb7..dcd803ac999 100644 --- a/agents/autowebcompat-intervention/Dockerfile +++ b/agents/autowebcompat-intervention/Dockerfile @@ -60,8 +60,8 @@ RUN cd /app && npm ci --omit=dev COPY agents/autowebcompat-intervention/hackbot.toml /app/hackbot.toml RUN useradd --create-home --shell /bin/bash agent \ - && mkdir -p /workspace /app/autowebcompat-intervention \ - && chown agent:agent /workspace /app/autowebcompat-intervention + && mkdir -p /workspace /home/agent/.mozbuild /app/autowebcompat-intervention \ + && chown agent:agent /workspace /home/agent/.mozbuild /app/autowebcompat-intervention ENV SCREEN_WIDTH=1280 ENV SCREEN_HEIGHT=1024 diff --git a/agents/autowebcompat-intervention/compose.yml b/agents/autowebcompat-intervention/compose.yml index 363eb7fe1b2..b562df31216 100644 --- a/agents/autowebcompat-intervention/compose.yml +++ b/agents/autowebcompat-intervention/compose.yml @@ -35,9 +35,11 @@ services: # The Firefox checkout and its objdir are large and slow to recreate; # persisting them across runs turns a cold artifact build into a warm one. - autowebcompat-intervention-workspace:/workspace + - mozbuild:/home/agent/.mozbuild depends_on: autowebcompat-intervention-broker: condition: service_started volumes: autowebcompat-intervention-workspace: + mozbuild: diff --git a/agents/bug-fix/Dockerfile b/agents/bug-fix/Dockerfile index e878bab9972..2dcd980111e 100644 --- a/agents/bug-fix/Dockerfile +++ b/agents/bug-fix/Dockerfile @@ -86,8 +86,8 @@ COPY agents/bug-fix/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN chmod +x /usr/local/bin/docker-entrypoint.sh RUN useradd --create-home --shell /bin/bash agent \ - && mkdir -p /workspace \ - && chown agent:agent /workspace \ + && mkdir -p /workspace /home/agent/.cargo /home/agent/.rustup /home/agent/.mozbuild \ + && chown agent:agent /workspace /home/agent/.cargo /home/agent/.rustup /home/agent/.mozbuild \ # Xvfb runs unprivileged and cannot create its own socket directory. && mkdir -p /tmp/.X11-unix \ && chmod 1777 /tmp/.X11-unix \ diff --git a/agents/bug-fix/compose.yml b/agents/bug-fix/compose.yml index e8c2e1cbce9..1df19577a7c 100644 --- a/agents/bug-fix/compose.yml +++ b/agents/bug-fix/compose.yml @@ -31,6 +31,9 @@ services: - ARTIFACTS_DIR=/artifacts volumes: - workspace:/workspace + - cargo:/home/agent/.cargo + - rustup:/home/agent/.rustup + - mozbuild:/home/agent/.mozbuild - ${HOME}/hackbot/artifacts:/artifacts depends_on: bug-fix-broker: @@ -38,3 +41,6 @@ services: volumes: workspace: + cargo: + rustup: + mozbuild: diff --git a/agents/bug-fix/hackbot_agents/bug_fix/agent.py b/agents/bug-fix/hackbot_agents/bug_fix/agent.py index da591a9c47d..3e27507dd29 100644 --- a/agents/bug-fix/hackbot_agents/bug_fix/agent.py +++ b/agents/bug-fix/hackbot_agents/bug_fix/agent.py @@ -170,6 +170,7 @@ async def run_bug_fix( # derives fx_ctx from the prepared source checkout and the agent's # hackbot.toml. Provision the config before the session can call build tools. _write_mozconfig(fx_ctx) + fx_ctx.bootstrap_before_build = True firefox_server = build_sdk_server("firefox", fx_ctx, firefox.TOOLS) user_prompt = select_workflow( diff --git a/agents/bug-fix/hackbot_agents/bug_fix/prompts/system.md b/agents/bug-fix/hackbot_agents/bug_fix/prompts/system.md index 9d507af8090..54f0af6bca0 100644 --- a/agents/bug-fix/hackbot_agents/bug_fix/prompts/system.md +++ b/agents/bug-fix/hackbot_agents/bug_fix/prompts/system.md @@ -29,8 +29,8 @@ Your working directory is the source repository for the product these bugs are f When a bug needs reproducing or fixing, you have `firefox` MCP tools: - `evaluate_testcase(content, filename, prefs?, timeout?)` — runs a testcase in Firefox under xvfb via grizzly (the build's sanitizer configuration depends on the configured mozconfig). Returns `crashed` (bool), `crashed_parent` (bool — parent-process crash), `logs` (stderr/stdout and, if crashed, `crashdata`), and the testcase bundle. When `crashed=false`, inspect `logs.stderr` for why (JS exception, gated pref, etc). -- `build_firefox()` — runs `./mach build` with the preconfigured mozconfig. Use it for every Firefox build instead of building or checking for a build from Bash. -- `bootstrap_firefox(application?)` — runs `./mach bootstrap` to install the build toolchain (rustc, cargo, clang, cbindgen). The image does not ship one, so if `build_firefox` reports a missing toolchain (e.g. no rustc or clang), call this once and then build again. Do not give up because a toolchain is missing. +- `build_firefox()` — prepares the native build toolchain before the first build, then runs `./mach build` with your mozconfig unchanged. Use it for every Firefox build instead of building or checking for a build from Bash. Pass `target` for a directory-only compile check; that does not produce a complete browser for running tests. +- `bootstrap_firefox(application?)` — explicitly changes which application to prepare. For a desktop artifact build, first select `application="browser_artifact_mode"` and use an artifact mozconfig; `build_firefox` then skips native toolchain preparation. Select `application="browser"` to return to native desktop builds. If preparation fails, inspect the returned error rather than treating it as a source-code failure. For Android bugs (GeckoView, Fenix, Focus, android-components, anything under `mobile/android`), call `bootstrap_firefox(application="mobile_android_artifact_mode")` (or `"mobile_android"` if you need to change C++ or Rust code), which installs a JDK and the Android SDK and NDK; `java`, `adb`, `aapt2`, `apksigner`, `apkanalyzer` and `lint` are then on PATH. It also writes a mozconfig for Android, which `build_firefox` then builds with; its result says where, and to run mach from Bash with it, e.g. `MOZCONFIG= ./mach gradle -p mobile/android/android-components :components:support-base:testDebugUnitTest` (use `./mach gradle` instead of `gradlew`, with `-p` for a subproject). JVM unit tests (Robolectric) work; there is no Android emulator or device, so instrumented tests can't run. diff --git a/agents/bug-fix/tests/test_build_setup.py b/agents/bug-fix/tests/test_build_setup.py new file mode 100644 index 00000000000..145320fc244 --- /dev/null +++ b/agents/bug-fix/tests/test_build_setup.py @@ -0,0 +1,38 @@ +"""Bug-fix opts into shared preparation without changing an existing config.""" + +import pytest +from agent_tools.firefox import FirefoxContext +from hackbot_agents.bug_fix import agent + + +@pytest.mark.parametrize("existing", [None, "ac_add_options --enable-js-shell\n"]) +async def test_build_preparation_is_enabled_before_the_session( + tmp_path, monkeypatch, existing +): + ctx = FirefoxContext.from_source_repo(tmp_path) + if existing is not None: + ctx.mozconfig.write_text(existing) + + def stop_before_session(name, context, tools): + assert name == "firefox" + assert context is ctx + assert context.bootstrap_before_build + assert context._bootstrapped is None + assert context._ci_build_env is None + if existing is not None: + assert context.mozconfig.read_text() == existing + else: + assert "--enable-debug" in context.mozconfig.read_text() + assert "--enable-clang-plugin" in context.mozconfig.read_text() + assert "--enable-js-shell" in context.mozconfig.read_text() + raise RuntimeError("stop before model session") + + monkeypatch.setattr(agent, "build_sdk_server", stop_before_session) + with pytest.raises(RuntimeError, match="stop before model session"): + await agent.run_bug_fix( + bugzilla_mcp_server={"type": "http", "url": "http://broker/bugzilla"}, + phabricator_mcp_server={"type": "http", "url": "http://broker/phabricator"}, + source_repo=tmp_path, + fx_ctx=ctx, + bug=1, + ) diff --git a/agents/build-repair/Dockerfile b/agents/build-repair/Dockerfile index b267424bcab..b684d44c287 100644 --- a/agents/build-repair/Dockerfile +++ b/agents/build-repair/Dockerfile @@ -55,8 +55,8 @@ RUN apt-get update \ COPY agents/build-repair/hackbot.toml /app/hackbot.toml RUN useradd --create-home --shell /bin/bash agent \ - && mkdir -p /workspace \ - && chown agent:agent /workspace + && mkdir -p /workspace /home/agent/.cargo /home/agent/.rustup /home/agent/.mozbuild \ + && chown agent:agent /workspace /home/agent/.cargo /home/agent/.rustup /home/agent/.mozbuild RUN mkdir /builds \ && for f in gls-gapi.data sb-gapi.data mozilla-desktop-geoloc-api.key; do echo no-secret > /builds/$f; done diff --git a/agents/build-repair/compose.yml b/agents/build-repair/compose.yml index c4904ac3277..1f7c13c2fa3 100644 --- a/agents/build-repair/compose.yml +++ b/agents/build-repair/compose.yml @@ -35,6 +35,9 @@ services: - ARTIFACTS_DIR=/artifacts volumes: - workspace:/workspace + - cargo:/home/agent/.cargo + - rustup:/home/agent/.rustup + - mozbuild:/home/agent/.mozbuild - ${HOME}/hackbot/artifacts:/artifacts depends_on: build-repair-broker: @@ -67,3 +70,6 @@ services: volumes: workspace: name: hackbot_workspace + cargo: + rustup: + mozbuild: diff --git a/docs/hackbot/tools.md b/docs/hackbot/tools.md index ba5aec70329..f4b9b711ad1 100644 --- a/docs/hackbot/tools.md +++ b/docs/hackbot/tools.md @@ -128,8 +128,36 @@ locally against the checkout. full tree, much less incrementally. An agent that only needs to confirm a localized fix compiles should pass `target` to build one directory instead of the whole tree. -For a CI reproduction, an agent with a failed Taskcluster task can configure the Firefox -context before starting the tool server: +### Native and artifact builds + +Agents keep ownership of their mozconfig. An agent that needs native builds can opt +into shared preparation before starting its tool server: + +```python +ctx.firefox.bootstrap_before_build = True +``` + +The first native `build_firefox` call runs the existing bootstrap tool before compiling. +It leaves the agent's mozconfig unchanged. Preparation and builds are serialized within +that context, including subagent calls. Failed preparation stops the build and can be +retried; source errors do not repeat bootstrap. Readiness lasts only for that context +and source directory. Use `bootstrap_firefox` explicitly after changing the checkout +within a session. + +Bug-fix opts in. Other agents do not gain automatic downloads, and triage-only runs do +not prepare anything. For a desktop artifact build, select +`bootstrap_firefox(application="browser_artifact_mode")` and supply an artifact +mozconfig; subsequent builds skip native bootstrap. Selecting `application="browser"` +returns to native desktop builds without changing the desktop mozconfig. Android keeps +its existing explicit application selection and separate generated mozconfig. + +A directory-only build is a compile check, not a complete browser for running tests. +An ignored-directory warning is a build failure even if mach exits successfully. + +### Reproducing a CI task + +An agent can explicitly configure the Firefox context from a Taskcluster build task +before starting the tool server: ```python from agent_tools.firefox import linux_build_env @@ -142,6 +170,18 @@ on x86-64 hosts. Call it once before building; later `build_firefox` calls reuse setup. It removes a stale in-tree mozconfig before fetching, so local runs can reuse the workspace volume. CI environment overrides apply only to build subprocesses. Unsupported tasks and non-x86-64 hosts keep the agent's normal build configuration. +This is the only shared setup path that replaces the desktop mozconfig. Configured CI +builds do not also run ordinary bootstrap. + +### Local cache reuse + +The Compose services for bug-fix and build-repair retain `.cargo`, `.rustup`, and +`.mozbuild` in named volumes alongside the workspace. Autowebcompat-intervention retains +`.mozbuild` for artifact downloads. Image directories belong to the agent so new volumes +are writable. These volumes do not mount the rest of the agent's home directory. + +This changes local Compose runs only. Cloud Run workspaces remain fresh; no remote +sccache backend, deployment resources, or production storage is changed. ## Adding a tool diff --git a/libs/agent-tools/agent_tools/firefox/__init__.py b/libs/agent-tools/agent_tools/firefox/__init__.py index 9853063da3d..067c94aebf1 100644 --- a/libs/agent-tools/agent_tools/firefox/__init__.py +++ b/libs/agent-tools/agent_tools/firefox/__init__.py @@ -5,6 +5,7 @@ first parameter and returns plain data. """ +import asyncio from dataclasses import dataclass, field from pathlib import Path from typing import Annotated, Literal @@ -37,7 +38,13 @@ class FirefoxContext: js_binary: Path # What bootstrap_firefox last set up, which build_firefox then builds for. application: str = "browser" + # Opt in without changing the agent's mozconfig or preparing triage-only runs. + bootstrap_before_build: bool = False _ci_build_env: dict[str, str] | None = field(default=None, init=False, repr=False) + _bootstrapped: tuple[Path, str] | None = field(default=None, init=False, repr=False) + _build_lock: asyncio.Lock = field( + default_factory=asyncio.Lock, init=False, repr=False + ) async def configure_ci_build(self, env: dict[str, str] | None) -> None: """Fetch a CI task's toolchains and use its mozconfig, or clear overrides.""" @@ -166,7 +173,11 @@ async def build_firefox( """Build Firefox using the configured mozconfig. For Android once ``bootstrap_firefox`` has been called for it, with the - mozconfig it wrote for that; for desktop Firefox otherwise. + mozconfig it wrote for that; for desktop Firefox otherwise. Agents can opt + into toolchain bootstrap before the first native build, without replacing + their mozconfig. Select desktop artifact mode with + ``bootstrap_firefox(application="browser_artifact_mode")`` before using an + artifact mozconfig; native bootstrap is then skipped. Slow on a full tree build (tens of minutes cold, faster incremental); pass a `target` directory to build just the part you changed. Returns JSON: success @@ -174,16 +185,30 @@ async def build_firefox( you've changed source or the binary is missing — check if the binary exists first. """ - firefox_dir_p = Path(firefox_dir) if firefox_dir else ctx.source_dir - mozconfig, objdir = ctx.build_config() - mozconfig_p = Path(mozconfig_path) if mozconfig_path else mozconfig - return await _build_firefox( - firefox_dir_p, - mozconfig_p, - objdir, - target=target, - env_overrides=ctx._ci_build_env, - ) + async with ctx._build_lock: + firefox_dir_p = Path(firefox_dir) if firefox_dir else ctx.source_dir + mozconfig, objdir = ctx.build_config() + mozconfig_p = Path(mozconfig_path) if mozconfig_path else mozconfig + bootstrap_key = (firefox_dir_p.resolve(), ctx.application) + if ( + ctx.bootstrap_before_build + and ctx._ci_build_env is None + and not ctx.application.endswith("_artifact_mode") + and ctx._bootstrapped != bootstrap_key + and firefox_dir_p.is_dir() + and mozconfig_p.is_file() + ): + result = await _bootstrap_firefox(firefox_dir_p, ctx.application) + if not result["success"]: + return result + ctx._bootstrapped = bootstrap_key + return await _build_firefox( + firefox_dir_p, + mozconfig_p, + objdir, + target=target, + env_overrides=ctx._ci_build_env, + ) @tool @@ -263,18 +288,21 @@ async def bootstrap_firefox( Bootstrapping for Android also writes a mozconfig for it, which ``build_firefox`` then uses; bootstrapping for desktop switches back. """ - firefox_dir_p = Path(firefox_dir) if firefox_dir else ctx.source_dir - result = await _bootstrap_firefox(firefox_dir_p, application) - if result["success"]: - ctx.application = application - if application.startswith("mobile_android"): - mozconfig, objdir = ctx.build_config() - mozconfig.write_text(_android_mozconfig(application, objdir)) - result["message"] += ( - f"; build_firefox now builds Firefox for Android, with {mozconfig} " - f"(for mach from Bash: MOZCONFIG={mozconfig} ./mach ...)" - ) - return result + async with ctx._build_lock: + firefox_dir_p = Path(firefox_dir) if firefox_dir else ctx.source_dir + ctx._bootstrapped = None + result = await _bootstrap_firefox(firefox_dir_p, application) + if result["success"]: + ctx.application = application + if application.startswith("mobile_android"): + mozconfig, objdir = ctx.build_config() + mozconfig.write_text(_android_mozconfig(application, objdir)) + result["message"] += ( + f"; build_firefox now builds Firefox for Android, with {mozconfig} " + f"(for mach from Bash: MOZCONFIG={mozconfig} ./mach ...)" + ) + ctx._bootstrapped = (firefox_dir_p.resolve(), application) + return result def _android_mozconfig(application: str, objdir: Path) -> str: diff --git a/libs/agent-tools/agent_tools/firefox/tools/build_firefox.py b/libs/agent-tools/agent_tools/firefox/tools/build_firefox.py index ab664ba7397..c5c0908974f 100644 --- a/libs/agent-tools/agent_tools/firefox/tools/build_firefox.py +++ b/libs/agent-tools/agent_tools/firefox/tools/build_firefox.py @@ -93,6 +93,18 @@ async def mach_build(*args: str) -> tuple[int | None, str, str]: else: returncode, stdout_output, stderr_output = await mach_build() + if ( + target + and returncode == 0 + and "is a subdirectory and was ignored" in stdout_output + stderr_output + ): + return { + "success": False, + "message": f"Firefox build skipped the requested target {target!r}", + "stdout": stdout_output, + "stderr": stderr_output, + } + if returncode == 0: return { "success": True, diff --git a/libs/agent-tools/tests/test_firefox.py b/libs/agent-tools/tests/test_firefox.py index 050ae35ecec..f46874e167e 100644 --- a/libs/agent-tools/tests/test_firefox.py +++ b/libs/agent-tools/tests/test_firefox.py @@ -2,6 +2,7 @@ import os +import pytest from agent_tools import firefox from agent_tools.claude_sdk import build_sdk_server from agent_tools.firefox.tools import bootstrap_firefox @@ -251,3 +252,39 @@ async def test_build_target_compiles_the_directory_after_its_prerequisites(tmp_p "build --quiet pre-export export", "build --quiet --allow-subdirectory-build build/clang-plugin image", ] + + +@pytest.mark.parametrize("stream", ["stdout", "stderr"]) +async def test_ignored_directory_is_not_a_successful_build(tmp_path, stream): + src = _recording_tree(tmp_path) + redirect = " >&2" if stream == "stderr" else "" + (src / "mach").write_text( + "#!/bin/sh\n" + 'if [ "$3" = --allow-subdirectory-build ]; then\n' + f' echo "Build argument image is a subdirectory and was ignored."{redirect}\n' + "fi\n" + "exit 0\n" + ) + config = src / ".mozconfig" + config.touch() + + result = await run_build(src, config, src / "objdir", target="image") + + assert not result["success"] + assert "skipped the requested target" in result["message"] + assert "is a subdirectory and was ignored" in result[stream] + + +async def test_failed_target_prerequisites_stop_the_build(tmp_path): + src = _recording_tree(tmp_path) + with (src / "mach").open("a") as mach: + mach.write("exit 2\n") + config = src / ".mozconfig" + config.touch() + + result = await run_build(src, config, src / "objdir", target="image") + + assert not result["success"] + assert (tmp_path / "mach.log").read_text().splitlines() == [ + "build --quiet pre-export export" + ] diff --git a/libs/agent-tools/tests/test_firefox_ci.py b/libs/agent-tools/tests/test_firefox_ci.py index 17afd294d7d..008f314788c 100644 --- a/libs/agent-tools/tests/test_firefox_ci.py +++ b/libs/agent-tools/tests/test_firefox_ci.py @@ -86,6 +86,7 @@ async def test_ci_build_setup_supports_reused_workspaces_without_changing_proces (src / ".mozconfig").write_text("left by the previous run\n") first = firefox.FirefoxContext.from_source_repo(src, objdir=str(objdir)) + first.bootstrap_before_build = True await first.configure_ci_build(ENV) assert (await firefox.build_firefox(first, target="image"))["success"] assert (await firefox.build_firefox(first, target="image"))["success"] diff --git a/libs/agent-tools/tests/test_firefox_preparation.py b/libs/agent-tools/tests/test_firefox_preparation.py new file mode 100644 index 00000000000..b0eb86a7592 --- /dev/null +++ b/libs/agent-tools/tests/test_firefox_preparation.py @@ -0,0 +1,157 @@ +"""Build preparation keeps the caller's configuration and is opt-in.""" + +import asyncio + +import pytest +from agent_tools import firefox + + +@pytest.fixture +def tree(tmp_path): + src = tmp_path / "firefox" + src.mkdir() + mach = src / "mach" + mach.write_text( + "#!/bin/sh\n" + f'echo "$*" >> "{tmp_path}/mach.log"\n' + 'if [ "$2" = bootstrap ]; then\n' + f' [ ! -e "{tmp_path}/fail-bootstrap" ] || exit 23\n' + "else\n" + f' [ ! -e "{tmp_path}/fail-build" ] || exit 2\n' + "fi\n" + ) + mach.chmod(0o755) + (src / ".mozconfig").write_text( + "ac_add_options --enable-debug\n" + "ac_add_options --enable-clang-plugin\n" + "ac_add_options --enable-js-shell\n" + ) + return src + + +def commands(tmp_path): + return (tmp_path / "mach.log").read_text().splitlines() + + +async def test_first_build_prepares_once_and_preserves_config_on_reused_workspace( + tree, tmp_path +): + config = (tree / ".mozconfig").read_bytes() + ctx = firefox.FirefoxContext.from_source_repo(tree) + ctx.bootstrap_before_build = True + assert not (tmp_path / "mach.log").exists() + + results = await asyncio.gather( + firefox.build_firefox(ctx), firefox.build_firefox(ctx) + ) + assert all(result["success"] for result in results) + assert commands(tmp_path) == [ + "--no-interactive bootstrap --application-choice=browser", + "build --quiet", + "build --quiet", + ] + + # Readiness is per run, not a persistent marker that can outlive the checkout. + next_run = firefox.FirefoxContext.from_source_repo(tree) + next_run.bootstrap_before_build = True + assert (await firefox.build_firefox(next_run))["success"] + assert commands(tmp_path)[-2:] == [ + "--no-interactive bootstrap --application-choice=browser", + "build --quiet", + ] + assert (tree / ".mozconfig").read_bytes() == config + assert not (tree / "fetches").exists() + + +async def test_build_does_not_prepare_without_opt_in(tree, tmp_path): + ctx = firefox.FirefoxContext.from_source_repo(tree) + assert (await firefox.build_firefox(ctx))["success"] + assert commands(tmp_path) == ["build --quiet"] + + +async def test_explicit_application_selection_avoids_duplicate_native_bootstrap( + tree, tmp_path +): + ctx = firefox.FirefoxContext.from_source_repo(tree) + ctx.bootstrap_before_build = True + config = ctx.mozconfig.read_bytes() + + result = await firefox.bootstrap_firefox(ctx, application="browser_artifact_mode") + assert result["success"] + assert (await firefox.build_firefox(ctx))["success"] + assert (await firefox.bootstrap_firefox(ctx, application="browser"))["success"] + assert (await firefox.build_firefox(ctx))["success"] + assert commands(tmp_path) == [ + "--no-interactive bootstrap --application-choice=browser_artifact_mode", + "build --quiet", + "--no-interactive bootstrap --application-choice=browser", + "build --quiet", + ] + assert ctx.mozconfig.read_bytes() == config + + +async def test_failed_preparation_stops_build_and_can_be_retried(tree, tmp_path): + ctx = firefox.FirefoxContext.from_source_repo(tree) + ctx.bootstrap_before_build = True + config = ctx.mozconfig.read_bytes() + failure = tmp_path / "fail-bootstrap" + failure.touch() + + result = await firefox.build_firefox(ctx) + assert not result["success"] + assert result["message"] == "mach bootstrap failed with exit code 23" + assert commands(tmp_path) == [ + "--no-interactive bootstrap --application-choice=browser" + ] + assert ctx.mozconfig.read_bytes() == config + + failure.unlink() + assert (await firefox.build_firefox(ctx))["success"] + assert commands(tmp_path)[-2:] == [ + "--no-interactive bootstrap --application-choice=browser", + "build --quiet", + ] + + +async def test_source_error_does_not_repeat_preparation(tree, tmp_path): + ctx = firefox.FirefoxContext.from_source_repo(tree) + ctx.bootstrap_before_build = True + (tmp_path / "fail-build").touch() + + for _ in range(2): + result = await firefox.build_firefox(ctx) + assert not result["success"] + assert result["message"] == "Firefox build failed with exit code 2" + assert commands(tmp_path) == [ + "--no-interactive bootstrap --application-choice=browser", + "build --quiet", + "build --quiet", + ] + + +async def test_missing_config_does_not_download_tools(tree, tmp_path): + ctx = firefox.FirefoxContext.from_source_repo(tree) + ctx.bootstrap_before_build = True + ctx.mozconfig.unlink() + + result = await firefox.build_firefox(ctx) + assert not result["success"] + assert result["message"].startswith("MOZCONFIG file not found") + assert not (tmp_path / "mach.log").exists() + + +async def test_source_override_gets_its_own_preparation(tree, tmp_path): + ctx = firefox.FirefoxContext.from_source_repo(tree) + ctx.bootstrap_before_build = True + assert (await firefox.build_firefox(ctx))["success"] + other = tmp_path / "other-firefox" + other.mkdir() + (other / "mach").symlink_to(tree / "mach") + + assert (await firefox.build_firefox(ctx, firefox_dir=str(other)))["success"] + assert commands(tmp_path) == [ + "--no-interactive bootstrap --application-choice=browser", + "build --quiet", + "--no-interactive bootstrap --application-choice=browser", + "build --quiet", + ] From 5e0973c9497bfd2ab8c597d6aa34b40d7b9c6d5f Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Tue, 3 Nov 2026 11:51:27 -0500 Subject: [PATCH 4/6] docs(firefox): simplify build preparation guidance --- docs/hackbot/tools.md | 48 +++++++++++++++---------------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/docs/hackbot/tools.md b/docs/hackbot/tools.md index f4b9b711ad1..19c62bce88d 100644 --- a/docs/hackbot/tools.md +++ b/docs/hackbot/tools.md @@ -130,34 +130,27 @@ compiles should pass `target` to build one directory instead of the whole tree. ### Native and artifact builds -Agents keep ownership of their mozconfig. An agent that needs native builds can opt -into shared preparation before starting its tool server: +Bug-fix opts into setup before its first native build: ```python ctx.firefox.bootstrap_before_build = True ``` -The first native `build_firefox` call runs the existing bootstrap tool before compiling. -It leaves the agent's mozconfig unchanged. Preparation and builds are serialized within -that context, including subagent calls. Failed preparation stops the build and can be -retried; source errors do not repeat bootstrap. Readiness lasts only for that context -and source directory. Use `bootstrap_firefox` explicitly after changing the checkout -within a session. +Bootstrap runs once per context and keeps the mozconfig unchanged. Setup failure stops +builds; triage-only runs do no setup. Other agents must opt in. After changing the +checkout within a session, run `bootstrap_firefox` again. -Bug-fix opts in. Other agents do not gain automatic downloads, and triage-only runs do -not prepare anything. For a desktop artifact build, select +For desktop artifact builds, select `bootstrap_firefox(application="browser_artifact_mode")` and supply an artifact -mozconfig; subsequent builds skip native bootstrap. Selecting `application="browser"` -returns to native desktop builds without changing the desktop mozconfig. Android keeps -its existing explicit application selection and separate generated mozconfig. +mozconfig. Use `application="browser"` to switch back. Android keeps its explicit +bootstrap selection and separate mozconfig. -A directory-only build is a compile check, not a complete browser for running tests. -An ignored-directory warning is a build failure even if mach exits successfully. +Directory builds check compilation; they do not produce a complete browser. +Ignored targets fail. ### Reproducing a CI task -An agent can explicitly configure the Firefox context from a Taskcluster build task -before starting the tool server: +Before building: ```python from agent_tools.firefox import linux_build_env @@ -165,23 +158,14 @@ from agent_tools.firefox import linux_build_env await ctx.firefox.configure_ci_build(linux_build_env(task_id)) ``` -This fetches the task's toolchains and writes its mozconfig for supported Linux tasks -on x86-64 hosts. Call it once before building; later `build_firefox` calls reuse the -setup. It removes a stale in-tree mozconfig before fetching, so local runs can reuse -the workspace volume. CI environment overrides apply only to build subprocesses. -Unsupported tasks and non-x86-64 hosts keep the agent's normal build configuration. -This is the only shared setup path that replaces the desktop mozconfig. Configured CI -builds do not also run ordinary bootstrap. +For supported Linux tasks on x86-64, this downloads the task's tools, replaces the +mozconfig, and skips ordinary bootstrap. Unsupported tasks leave the config unchanged. -### Local cache reuse +### Local caches -The Compose services for bug-fix and build-repair retain `.cargo`, `.rustup`, and -`.mozbuild` in named volumes alongside the workspace. Autowebcompat-intervention retains -`.mozbuild` for artifact downloads. Image directories belong to the agent so new volumes -are writable. These volumes do not mount the rest of the agent's home directory. - -This changes local Compose runs only. Cloud Run workspaces remain fresh; no remote -sccache backend, deployment resources, or production storage is changed. +Compose retains `.cargo`, `.rustup`, and `.mozbuild` for bug-fix and build-repair, and +`.mozbuild` for autowebcompat-intervention—not the whole home directory. These mounts +do not change Cloud Run storage or resources, or configure remote sccache. ## Adding a tool From 51761d402834a3c95906a5b6e6cb389e4e23dcd7 Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Thu, 19 Nov 2026 15:16:39 -0500 Subject: [PATCH 5/6] refactor(firefox): simplify build preparation --- .../hackbot_agents/bug_fix/prompts/system.md | 4 +- agents/bug-fix/tests/test_build_setup.py | 38 ---------------- .../hackbot_agents/build_repair/prompts.py | 6 +-- docs/hackbot/tools.md | 42 +++--------------- .../agent_tools/firefox/__init__.py | 37 +++++----------- libs/agent-tools/tests/test_firefox_ci.py | 2 - .../tests/test_firefox_preparation.py | 44 +------------------ 7 files changed, 24 insertions(+), 149 deletions(-) delete mode 100644 agents/bug-fix/tests/test_build_setup.py diff --git a/agents/bug-fix/hackbot_agents/bug_fix/prompts/system.md b/agents/bug-fix/hackbot_agents/bug_fix/prompts/system.md index 54f0af6bca0..9e2100229f3 100644 --- a/agents/bug-fix/hackbot_agents/bug_fix/prompts/system.md +++ b/agents/bug-fix/hackbot_agents/bug_fix/prompts/system.md @@ -29,8 +29,8 @@ Your working directory is the source repository for the product these bugs are f When a bug needs reproducing or fixing, you have `firefox` MCP tools: - `evaluate_testcase(content, filename, prefs?, timeout?)` — runs a testcase in Firefox under xvfb via grizzly (the build's sanitizer configuration depends on the configured mozconfig). Returns `crashed` (bool), `crashed_parent` (bool — parent-process crash), `logs` (stderr/stdout and, if crashed, `crashdata`), and the testcase bundle. When `crashed=false`, inspect `logs.stderr` for why (JS exception, gated pref, etc). -- `build_firefox()` — prepares the native build toolchain before the first build, then runs `./mach build` with your mozconfig unchanged. Use it for every Firefox build instead of building or checking for a build from Bash. Pass `target` for a directory-only compile check; that does not produce a complete browser for running tests. -- `bootstrap_firefox(application?)` — explicitly changes which application to prepare. For a desktop artifact build, first select `application="browser_artifact_mode"` and use an artifact mozconfig; `build_firefox` then skips native toolchain preparation. Select `application="browser"` to return to native desktop builds. If preparation fails, inspect the returned error rather than treating it as a source-code failure. +- `build_firefox(target?)` — runs `./mach build` with the preconfigured mozconfig, after a one-time `mach bootstrap`. Use it for every Firefox build instead of building from Bash. Pass `target` (a directory) for a quick compile check. +- `bootstrap_firefox(application?)` — sets up the toolchain for another application, e.g. `browser_artifact_mode` for an artifact build. For Android bugs (GeckoView, Fenix, Focus, android-components, anything under `mobile/android`), call `bootstrap_firefox(application="mobile_android_artifact_mode")` (or `"mobile_android"` if you need to change C++ or Rust code), which installs a JDK and the Android SDK and NDK; `java`, `adb`, `aapt2`, `apksigner`, `apkanalyzer` and `lint` are then on PATH. It also writes a mozconfig for Android, which `build_firefox` then builds with; its result says where, and to run mach from Bash with it, e.g. `MOZCONFIG= ./mach gradle -p mobile/android/android-components :components:support-base:testDebugUnitTest` (use `./mach gradle` instead of `gradlew`, with `-p` for a subproject). JVM unit tests (Robolectric) work; there is no Android emulator or device, so instrumented tests can't run. diff --git a/agents/bug-fix/tests/test_build_setup.py b/agents/bug-fix/tests/test_build_setup.py deleted file mode 100644 index 145320fc244..00000000000 --- a/agents/bug-fix/tests/test_build_setup.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Bug-fix opts into shared preparation without changing an existing config.""" - -import pytest -from agent_tools.firefox import FirefoxContext -from hackbot_agents.bug_fix import agent - - -@pytest.mark.parametrize("existing", [None, "ac_add_options --enable-js-shell\n"]) -async def test_build_preparation_is_enabled_before_the_session( - tmp_path, monkeypatch, existing -): - ctx = FirefoxContext.from_source_repo(tmp_path) - if existing is not None: - ctx.mozconfig.write_text(existing) - - def stop_before_session(name, context, tools): - assert name == "firefox" - assert context is ctx - assert context.bootstrap_before_build - assert context._bootstrapped is None - assert context._ci_build_env is None - if existing is not None: - assert context.mozconfig.read_text() == existing - else: - assert "--enable-debug" in context.mozconfig.read_text() - assert "--enable-clang-plugin" in context.mozconfig.read_text() - assert "--enable-js-shell" in context.mozconfig.read_text() - raise RuntimeError("stop before model session") - - monkeypatch.setattr(agent, "build_sdk_server", stop_before_session) - with pytest.raises(RuntimeError, match="stop before model session"): - await agent.run_bug_fix( - bugzilla_mcp_server={"type": "http", "url": "http://broker/bugzilla"}, - phabricator_mcp_server={"type": "http", "url": "http://broker/phabricator"}, - source_repo=tmp_path, - fx_ctx=ctx, - bug=1, - ) diff --git a/agents/build-repair/hackbot_agents/build_repair/prompts.py b/agents/build-repair/hackbot_agents/build_repair/prompts.py index 7ad098fda6e..117407a03dd 100644 --- a/agents/build-repair/hackbot_agents/build_repair/prompts.py +++ b/agents/build-repair/hackbot_agents/build_repair/prompts.py @@ -173,10 +173,8 @@ Logs already fetched sit under {scratch_out}/logs. The mozconfig and any supported CI toolchains are already set up. Verify the fix compiles with build_firefox, passing the directory of the file you changed as `target` (e.g. 'docshell/base') for a -fast, focused build -- prefer this over a full tree build. If using the CI -configuration, do not replace its toolchains with bootstrap; report a setup -failure. For the default configuration, bootstrap once if the build reports a -missing toolchain. Verify via the build_firefox tool rather than a raw `./mach +fast, focused build -- prefer this over a full tree build. Do not run +bootstrap_firefox over CI toolchains. Verify via the build_firefox tool rather than a raw `./mach build` so the build result is recorded. {try_push}{report}""" diff --git a/docs/hackbot/tools.md b/docs/hackbot/tools.md index 19c62bce88d..df71a6d7301 100644 --- a/docs/hackbot/tools.md +++ b/docs/hackbot/tools.md @@ -128,44 +128,14 @@ locally against the checkout. full tree, much less incrementally. An agent that only needs to confirm a localized fix compiles should pass `target` to build one directory instead of the whole tree. -### Native and artifact builds +### Bootstrap and CI builds -Bug-fix opts into setup before its first native build: +Set `ctx.firefox.bootstrap_before_build = True` to run `mach bootstrap` once before +the first build. Bug-fix does this. An explicit `bootstrap_firefox` call also counts. -```python -ctx.firefox.bootstrap_before_build = True -``` - -Bootstrap runs once per context and keeps the mozconfig unchanged. Setup failure stops -builds; triage-only runs do no setup. Other agents must opt in. After changing the -checkout within a session, run `bootstrap_firefox` again. - -For desktop artifact builds, select -`bootstrap_firefox(application="browser_artifact_mode")` and supply an artifact -mozconfig. Use `application="browser"` to switch back. Android keeps its explicit -bootstrap selection and separate mozconfig. - -Directory builds check compilation; they do not produce a complete browser. -Ignored targets fail. - -### Reproducing a CI task - -Before building: - -```python -from agent_tools.firefox import linux_build_env - -await ctx.firefox.configure_ci_build(linux_build_env(task_id)) -``` - -For supported Linux tasks on x86-64, this downloads the task's tools, replaces the -mozconfig, and skips ordinary bootstrap. Unsupported tasks leave the config unchanged. - -### Local caches - -Compose retains `.cargo`, `.rustup`, and `.mozbuild` for bug-fix and build-repair, and -`.mozbuild` for autowebcompat-intervention—not the whole home directory. These mounts -do not change Cloud Run storage or resources, or configure remote sccache. +To build as a Linux CI task did, call +`await ctx.firefox.configure_ci_build(linux_build_env(task_id))` before building. +It downloads the task's toolchains and uses its mozconfig. ## Adding a tool diff --git a/libs/agent-tools/agent_tools/firefox/__init__.py b/libs/agent-tools/agent_tools/firefox/__init__.py index 067c94aebf1..d28646aa2d8 100644 --- a/libs/agent-tools/agent_tools/firefox/__init__.py +++ b/libs/agent-tools/agent_tools/firefox/__init__.py @@ -38,21 +38,21 @@ class FirefoxContext: js_binary: Path # What bootstrap_firefox last set up, which build_firefox then builds for. application: str = "browser" - # Opt in without changing the agent's mozconfig or preparing triage-only runs. + # Bootstrap before the first build, so triage-only runs skip it. bootstrap_before_build: bool = False _ci_build_env: dict[str, str] | None = field(default=None, init=False, repr=False) - _bootstrapped: tuple[Path, str] | None = field(default=None, init=False, repr=False) + _bootstrapped: bool = field(default=False, init=False, repr=False) _build_lock: asyncio.Lock = field( default_factory=asyncio.Lock, init=False, repr=False ) - async def configure_ci_build(self, env: dict[str, str] | None) -> None: - """Fetch a CI task's toolchains and use its mozconfig, or clear overrides.""" - self._ci_build_env = ( - await prepare_ci_build(self.source_dir, self.mozconfig, self.objdir, env) - if env is not None - else None + async def configure_ci_build(self, env: dict[str, str]) -> None: + """Fetch a CI task's toolchains and use its mozconfig.""" + self._ci_build_env = await prepare_ci_build( + self.source_dir, self.mozconfig, self.objdir, env ) + # CI's toolchains replace bootstrap; bootstrapping over them would break the build. + self._bootstrapped = True @property def android_mozconfig(self) -> Path: @@ -173,11 +173,7 @@ async def build_firefox( """Build Firefox using the configured mozconfig. For Android once ``bootstrap_firefox`` has been called for it, with the - mozconfig it wrote for that; for desktop Firefox otherwise. Agents can opt - into toolchain bootstrap before the first native build, without replacing - their mozconfig. Select desktop artifact mode with - ``bootstrap_firefox(application="browser_artifact_mode")`` before using an - artifact mozconfig; native bootstrap is then skipped. + mozconfig it wrote for that; for desktop Firefox otherwise. Slow on a full tree build (tens of minutes cold, faster incremental); pass a `target` directory to build just the part you changed. Returns JSON: success @@ -189,19 +185,11 @@ async def build_firefox( firefox_dir_p = Path(firefox_dir) if firefox_dir else ctx.source_dir mozconfig, objdir = ctx.build_config() mozconfig_p = Path(mozconfig_path) if mozconfig_path else mozconfig - bootstrap_key = (firefox_dir_p.resolve(), ctx.application) - if ( - ctx.bootstrap_before_build - and ctx._ci_build_env is None - and not ctx.application.endswith("_artifact_mode") - and ctx._bootstrapped != bootstrap_key - and firefox_dir_p.is_dir() - and mozconfig_p.is_file() - ): + if ctx.bootstrap_before_build and not ctx._bootstrapped: result = await _bootstrap_firefox(firefox_dir_p, ctx.application) if not result["success"]: return result - ctx._bootstrapped = bootstrap_key + ctx._bootstrapped = True return await _build_firefox( firefox_dir_p, mozconfig_p, @@ -290,10 +278,10 @@ async def bootstrap_firefox( """ async with ctx._build_lock: firefox_dir_p = Path(firefox_dir) if firefox_dir else ctx.source_dir - ctx._bootstrapped = None result = await _bootstrap_firefox(firefox_dir_p, application) if result["success"]: ctx.application = application + ctx._bootstrapped = True if application.startswith("mobile_android"): mozconfig, objdir = ctx.build_config() mozconfig.write_text(_android_mozconfig(application, objdir)) @@ -301,7 +289,6 @@ async def bootstrap_firefox( f"; build_firefox now builds Firefox for Android, with {mozconfig} " f"(for mach from Bash: MOZCONFIG={mozconfig} ./mach ...)" ) - ctx._bootstrapped = (firefox_dir_p.resolve(), application) return result diff --git a/libs/agent-tools/tests/test_firefox_ci.py b/libs/agent-tools/tests/test_firefox_ci.py index 008f314788c..4b39c01c1fc 100644 --- a/libs/agent-tools/tests/test_firefox_ci.py +++ b/libs/agent-tools/tests/test_firefox_ci.py @@ -117,8 +117,6 @@ async def test_ci_build_setup_supports_reused_workspaces_without_changing_proces '. "$topsrcdir/browser/config/mozconfigs/linux64/nightly"', f"mk_add_options MOZ_OBJDIR={objdir}", ] - await first.configure_ci_build(None) - assert first._ci_build_env is None async def test_failed_ci_setup_does_not_write_a_build_mozconfig(tmp_path, monkeypatch): diff --git a/libs/agent-tools/tests/test_firefox_preparation.py b/libs/agent-tools/tests/test_firefox_preparation.py index b0eb86a7592..5d7fc6cf305 100644 --- a/libs/agent-tools/tests/test_firefox_preparation.py +++ b/libs/agent-tools/tests/test_firefox_preparation.py @@ -1,4 +1,4 @@ -"""Build preparation keeps the caller's configuration and is opt-in.""" +"""Opt-in bootstrap before the first build.""" import asyncio @@ -33,9 +33,7 @@ def commands(tmp_path): return (tmp_path / "mach.log").read_text().splitlines() -async def test_first_build_prepares_once_and_preserves_config_on_reused_workspace( - tree, tmp_path -): +async def test_first_build_prepares_once_and_keeps_the_mozconfig(tree, tmp_path): config = (tree / ".mozconfig").read_bytes() ctx = firefox.FirefoxContext.from_source_repo(tree) ctx.bootstrap_before_build = True @@ -50,17 +48,7 @@ async def test_first_build_prepares_once_and_preserves_config_on_reused_workspac "build --quiet", "build --quiet", ] - - # Readiness is per run, not a persistent marker that can outlive the checkout. - next_run = firefox.FirefoxContext.from_source_repo(tree) - next_run.bootstrap_before_build = True - assert (await firefox.build_firefox(next_run))["success"] - assert commands(tmp_path)[-2:] == [ - "--no-interactive bootstrap --application-choice=browser", - "build --quiet", - ] assert (tree / ".mozconfig").read_bytes() == config - assert not (tree / "fetches").exists() async def test_build_does_not_prepare_without_opt_in(tree, tmp_path): @@ -127,31 +115,3 @@ async def test_source_error_does_not_repeat_preparation(tree, tmp_path): "build --quiet", "build --quiet", ] - - -async def test_missing_config_does_not_download_tools(tree, tmp_path): - ctx = firefox.FirefoxContext.from_source_repo(tree) - ctx.bootstrap_before_build = True - ctx.mozconfig.unlink() - - result = await firefox.build_firefox(ctx) - assert not result["success"] - assert result["message"].startswith("MOZCONFIG file not found") - assert not (tmp_path / "mach.log").exists() - - -async def test_source_override_gets_its_own_preparation(tree, tmp_path): - ctx = firefox.FirefoxContext.from_source_repo(tree) - ctx.bootstrap_before_build = True - assert (await firefox.build_firefox(ctx))["success"] - other = tmp_path / "other-firefox" - other.mkdir() - (other / "mach").symlink_to(tree / "mach") - - assert (await firefox.build_firefox(ctx, firefox_dir=str(other)))["success"] - assert commands(tmp_path) == [ - "--no-interactive bootstrap --application-choice=browser", - "build --quiet", - "--no-interactive bootstrap --application-choice=browser", - "build --quiet", - ] From bbbc19bde4349c8471bfcb803c3715be8c4f1757 Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Thu, 19 Nov 2026 15:18:09 -0500 Subject: [PATCH 6/6] revert(agents): leave local cache volumes out of this change --- agents/autowebcompat-intervention/Dockerfile | 4 ++-- agents/autowebcompat-intervention/compose.yml | 2 -- agents/bug-fix/Dockerfile | 4 ++-- agents/bug-fix/compose.yml | 6 ------ agents/build-repair/Dockerfile | 4 ++-- agents/build-repair/compose.yml | 6 ------ 6 files changed, 6 insertions(+), 20 deletions(-) diff --git a/agents/autowebcompat-intervention/Dockerfile b/agents/autowebcompat-intervention/Dockerfile index dcd803ac999..16219bedcb7 100644 --- a/agents/autowebcompat-intervention/Dockerfile +++ b/agents/autowebcompat-intervention/Dockerfile @@ -60,8 +60,8 @@ RUN cd /app && npm ci --omit=dev COPY agents/autowebcompat-intervention/hackbot.toml /app/hackbot.toml RUN useradd --create-home --shell /bin/bash agent \ - && mkdir -p /workspace /home/agent/.mozbuild /app/autowebcompat-intervention \ - && chown agent:agent /workspace /home/agent/.mozbuild /app/autowebcompat-intervention + && mkdir -p /workspace /app/autowebcompat-intervention \ + && chown agent:agent /workspace /app/autowebcompat-intervention ENV SCREEN_WIDTH=1280 ENV SCREEN_HEIGHT=1024 diff --git a/agents/autowebcompat-intervention/compose.yml b/agents/autowebcompat-intervention/compose.yml index b562df31216..363eb7fe1b2 100644 --- a/agents/autowebcompat-intervention/compose.yml +++ b/agents/autowebcompat-intervention/compose.yml @@ -35,11 +35,9 @@ services: # The Firefox checkout and its objdir are large and slow to recreate; # persisting them across runs turns a cold artifact build into a warm one. - autowebcompat-intervention-workspace:/workspace - - mozbuild:/home/agent/.mozbuild depends_on: autowebcompat-intervention-broker: condition: service_started volumes: autowebcompat-intervention-workspace: - mozbuild: diff --git a/agents/bug-fix/Dockerfile b/agents/bug-fix/Dockerfile index 2dcd980111e..e878bab9972 100644 --- a/agents/bug-fix/Dockerfile +++ b/agents/bug-fix/Dockerfile @@ -86,8 +86,8 @@ COPY agents/bug-fix/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN chmod +x /usr/local/bin/docker-entrypoint.sh RUN useradd --create-home --shell /bin/bash agent \ - && mkdir -p /workspace /home/agent/.cargo /home/agent/.rustup /home/agent/.mozbuild \ - && chown agent:agent /workspace /home/agent/.cargo /home/agent/.rustup /home/agent/.mozbuild \ + && mkdir -p /workspace \ + && chown agent:agent /workspace \ # Xvfb runs unprivileged and cannot create its own socket directory. && mkdir -p /tmp/.X11-unix \ && chmod 1777 /tmp/.X11-unix \ diff --git a/agents/bug-fix/compose.yml b/agents/bug-fix/compose.yml index 1df19577a7c..e8c2e1cbce9 100644 --- a/agents/bug-fix/compose.yml +++ b/agents/bug-fix/compose.yml @@ -31,9 +31,6 @@ services: - ARTIFACTS_DIR=/artifacts volumes: - workspace:/workspace - - cargo:/home/agent/.cargo - - rustup:/home/agent/.rustup - - mozbuild:/home/agent/.mozbuild - ${HOME}/hackbot/artifacts:/artifacts depends_on: bug-fix-broker: @@ -41,6 +38,3 @@ services: volumes: workspace: - cargo: - rustup: - mozbuild: diff --git a/agents/build-repair/Dockerfile b/agents/build-repair/Dockerfile index b684d44c287..b267424bcab 100644 --- a/agents/build-repair/Dockerfile +++ b/agents/build-repair/Dockerfile @@ -55,8 +55,8 @@ RUN apt-get update \ COPY agents/build-repair/hackbot.toml /app/hackbot.toml RUN useradd --create-home --shell /bin/bash agent \ - && mkdir -p /workspace /home/agent/.cargo /home/agent/.rustup /home/agent/.mozbuild \ - && chown agent:agent /workspace /home/agent/.cargo /home/agent/.rustup /home/agent/.mozbuild + && mkdir -p /workspace \ + && chown agent:agent /workspace RUN mkdir /builds \ && for f in gls-gapi.data sb-gapi.data mozilla-desktop-geoloc-api.key; do echo no-secret > /builds/$f; done diff --git a/agents/build-repair/compose.yml b/agents/build-repair/compose.yml index 1f7c13c2fa3..c4904ac3277 100644 --- a/agents/build-repair/compose.yml +++ b/agents/build-repair/compose.yml @@ -35,9 +35,6 @@ services: - ARTIFACTS_DIR=/artifacts volumes: - workspace:/workspace - - cargo:/home/agent/.cargo - - rustup:/home/agent/.rustup - - mozbuild:/home/agent/.mozbuild - ${HOME}/hackbot/artifacts:/artifacts depends_on: build-repair-broker: @@ -70,6 +67,3 @@ services: volumes: workspace: name: hackbot_workspace - cargo: - rustup: - mozbuild: