From e4d9d50b0e0007b31a9dbb4c3937b301e750c25b Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 01:32:10 -0400 Subject: [PATCH 01/41] Add isolated native Windows voice runtime diagnostic (Refs #62) --- .../test_voice_runtime_diagnostic_workflow.py | 73 +++++ .github/workflows/bazel.yml | 74 ++++- codex-rs/voice-host/BUILD.bazel | 15 + .../voice/test_windows_runtime_diagnostic.py | 157 ++++++++++ .../voice/windows_runtime_diagnostic.bzl | 53 ++++ .../voice/windows_runtime_diagnostic.py | 292 ++++++++++++++++++ 6 files changed, 661 insertions(+), 3 deletions(-) create mode 100644 .github/scripts/test_voice_runtime_diagnostic_workflow.py create mode 100644 third_party/voice/test_windows_runtime_diagnostic.py create mode 100644 third_party/voice/windows_runtime_diagnostic.bzl create mode 100644 third_party/voice/windows_runtime_diagnostic.py diff --git a/.github/scripts/test_voice_runtime_diagnostic_workflow.py b/.github/scripts/test_voice_runtime_diagnostic_workflow.py new file mode 100644 index 000000000000..24e095553633 --- /dev/null +++ b/.github/scripts/test_voice_runtime_diagnostic_workflow.py @@ -0,0 +1,73 @@ +"""Structural checks for the deliberately isolated manual diagnostic route.""" + +from pathlib import Path +import re +import unittest + + +class WorkflowTests(unittest.TestCase): + def setUp(self): + self.text = ( + Path(__file__).resolve().parents[1] / "workflows/bazel.yml" + ).read_text() + self.jobs = dict( + re.findall( + r"^ ([\w-]+):\n(.*?)(?=^ [\w-]+:|\Z)", + self.text.split("jobs:\n", 1)[1], + re.M | re.S, + ) + ) + + def test_all_normal_jobs_guarded_and_original_predicates_retained(self): + ordinary = { + "test", + "test-windows-shard", + "test-windows", + "test-windows-native-main", + "clippy", + "verify-release-build", + } + self.assertEqual(set(self.jobs), ordinary | {"voice-runtime-diagnostic"}) + for name in ordinary: + gate = re.search(r"^ if: (.+)$", self.jobs[name], re.M).group(1) + self.assertIn( + "!(github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic)", + gate, + ) + # The added conjunction is true for normal dispatch, call, PR, push; + # false only for manual true mode, preserving prior predicates. + for event in ["workflow_dispatch", "workflow_call", "pull_request", "push"]: + for value in [False, None, True]: + self.assertEqual( + not (event == "workflow_dispatch" and value), + not (event == "workflow_dispatch" and value is True), + ) + self.assertIn("if: always() && !(", self.jobs["test-windows"]) + self.assertIn( + "github.event_name == 'push' && github.ref == 'refs/heads/main'", + self.jobs["test-windows-native-main"], + ) + + def test_diagnostic_scope_and_credentials(self): + job = self.jobs["voice-runtime-diagnostic"] + self.assertIn( + "if: github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic", + job, + ) + self.assertIn("contents: read", job) + self.assertNotIn("secrets.", job) + self.assertNotIn("write", job) + self.assertIn("unset BUILDBUDDY_API_KEY", job) + self.assertIn("persist-credentials: false", job) + self.assertIn("native-windows-voice: true", job) + self.assertIn("--nocache_test_results", job) + self.assertEqual( + re.findall(r"//codex-rs/[^\s]+", job), + ["//codex-rs/voice-host:voice-host-windows-runtime-diagnostic-test"], + ) + self.assertNotIn("//...", job) + self.assertIn("'voice-diagnostic' || 'normal'", self.text.split("jobs:", 1)[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index c33dafe85734..b39691f99085 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -6,15 +6,21 @@ name: Bazel on: workflow_call: workflow_dispatch: + inputs: + voice_runtime_diagnostic: + description: Run only the bounded native MSVC voice loader diagnostic + type: boolean + default: false concurrency: # Cancel previous actions from the same PR or branch except 'main' branch. # See https://docs.github.com/en/actions/using-jobs/using-concurrency and https://docs.github.com/en/actions/learn-github-actions/contexts for more info. - group: concurrency-group::${{ github.workflow }}::${{ github.event.pull_request.number > 0 && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}${{ github.ref_name == 'main' && format('::{0}', github.run_id) || ''}} + group: concurrency-group::${{ github.workflow }}::${{ github.event.pull_request.number > 0 && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}${{ github.ref_name == 'main' && format('::{0}', github.run_id) || ''}}::${{ github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic && 'voice-diagnostic' || 'normal' }} cancel-in-progress: ${{ github.ref_name != 'main' }} jobs: test: + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic) }} # PRs use the sharded Windows cross-compiled test jobs below. Post-merge # pushes to main also run the native Windows test job for broader Windows # signal without putting PR latency back on the critical path. When @@ -140,6 +146,7 @@ jobs: uses: ./.github/actions/check-clean-worktree test-windows-shard: + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic) }} # Split the Windows Bazel test leg across separate Windows hosts. Jobs with # BuildBuddy credentials use Linux RBE for build actions; test execution # remains on a Windows runner. @@ -248,7 +255,7 @@ jobs: test-windows: # Preserve the existing required-check surface while the real work happens # in the sharded Windows jobs above. - if: always() + if: always() && !(github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic) needs: test-windows-shard runs-on: ubuntu-24.04 name: Bazel test on windows-latest for x86_64-pc-windows-gnullvm @@ -266,7 +273,7 @@ jobs: # Native Windows Bazel tests are slower and frequently approach the # 30-minute PR budget. Run this only for post-merge commits to main and give # it a larger timeout. Also allow manual runs for native toolchain changes. - if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'workflow_dispatch' + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic) && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'workflow_dispatch') }} timeout-minutes: 40 runs-on: ${{ github.repository == 'openai/codex' && fromJSON(format('{{"group":"{0}-runners","labels":"{0}-windows-x64"}}', github.event.repository.name)) || 'windows-2025' }} name: Bazel test on windows-latest for x86_64-pc-windows-msvc (native main) @@ -348,6 +355,7 @@ jobs: uses: ./.github/actions/check-clean-worktree clippy: + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic) }} timeout-minutes: ${{ github.repository == 'openai/codex' && 30 || 90 }} strategy: fail-fast: false @@ -465,6 +473,7 @@ jobs: uses: ./.github/actions/check-clean-worktree verify-release-build: + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic) }} timeout-minutes: ${{ github.repository == 'openai/codex' && 30 || 90 }} strategy: fail-fast: false @@ -593,3 +602,62 @@ jobs: - name: Check for a clean worktree if: always() && !cancelled() uses: ./.github/actions/check-clean-worktree + + voice-runtime-diagnostic: + if: github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic + runs-on: windows-2025 + timeout-minutes: 60 + permissions: + contents: read + environment: + name: bazel + deployment: false + steps: + - name: Enable long paths for Windows checkout + shell: pwsh + working-directory: ${{ runner.temp }} + run: git config --global core.longpaths true + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Record actual checkout provenance + shell: pwsh + run: | + "VOICE_DIAGNOSTIC_HEAD=$(git rev-parse HEAD)" >> $env:GITHUB_ENV + "VOICE_DIAGNOSTIC_TREE=$(git rev-parse 'HEAD^{tree}')" >> $env:GITHUB_ENV + - name: Prepare Bazel CI + uses: ./.github/actions/prepare-bazel-ci + with: + target: x86_64-pc-windows-gnullvm + native-windows-voice: true + cache-scope: bazel-test + - name: Diagnose native MSVC voice runtime + timeout-minutes: 45 + shell: bash + # Deliberately no BuildBuddy credential: the existing wrapper selects + # its native MSVC route. This is not GNU cross-compile evidence. + run: | + unset BUILDBUDDY_API_KEY + ./.github/scripts/run-bazel-ci.sh \ + --print-failed-test-logs --print-failed-action-summary \ + --windows-cross-compile -- test --nocache_test_results --test_output=all --zip_undeclared_test_outputs=false \ + --test_env=VOICE_DIAGNOSTIC_HEAD --test_env=VOICE_DIAGNOSTIC_TREE \ + --test_env=GITHUB_SHA --test_env=GITHUB_EVENT_NAME --test_env=GITHUB_REF --test_env=GITHUB_RUN_ID \ + --build_metadata=COMMIT_SHA=${GITHUB_SHA} -- \ + //codex-rs/voice-host:voice-host-windows-runtime-diagnostic-test + - name: Collect diagnostic text outputs + if: always() && !cancelled() + shell: bash + run: | + mkdir -p "${RUNNER_TEMP}/voice-runtime-diagnostic" + # Copy only the named text result/log, never executables or runfiles. + for file in bazel-testlogs/codex-rs/voice-host/voice-host-windows-runtime-diagnostic-test/test.outputs/voice-runtime-diagnostic.json bazel-testlogs/codex-rs/voice-host/voice-host-windows-runtime-diagnostic-test/test.log; do + if [[ -f "$file" ]]; then cp "$file" "${RUNNER_TEMP}/voice-runtime-diagnostic/"; fi + done + - name: Upload diagnostic text evidence + if: always() && !cancelled() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: voice-runtime-diagnostic-${{ github.sha }} + path: ${{ runner.temp }}/voice-runtime-diagnostic + if-no-files-found: warn diff --git a/codex-rs/voice-host/BUILD.bazel b/codex-rs/voice-host/BUILD.bazel index 665141e70651..5ee6bd24d195 100644 --- a/codex-rs/voice-host/BUILD.bazel +++ b/codex-rs/voice-host/BUILD.bazel @@ -1,4 +1,5 @@ load("//:defs.bzl", "codex_rust_crate") +load("//third_party/voice:windows_runtime_diagnostic.bzl", "windows_runtime_diagnostic_test") codex_rust_crate( name = "voice-host", @@ -7,3 +8,17 @@ codex_rust_crate( crate_srcs = [], unit_test_args = ["--test-threads=1"], ) + +# Explicit observational target: ordinary //... tests must retain their own gates. +windows_runtime_diagnostic_test( + name = "voice-host-windows-runtime-diagnostic-test", + timeout = "moderate", + helper = ":codex-voice-host", + tags = ["manual"], + target_compatible_with = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + "@llvm//constraints/windows/abi:msvc", + ], + unit = ":codex-voice-host-bin-unit-tests-bin", +) diff --git a/third_party/voice/test_windows_runtime_diagnostic.py b/third_party/voice/test_windows_runtime_diagnostic.py new file mode 100644 index 000000000000..4a59f886c813 --- /dev/null +++ b/third_party/voice/test_windows_runtime_diagnostic.py @@ -0,0 +1,157 @@ +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import windows_runtime_diagnostic as diagnostic + + +class DiagnosticTests(unittest.TestCase): + def test_exe_dll_and_deferred_imports(self): + text = """File Type: EXECUTABLE IMAGE + 8664 machine (x64) + Image has the following dependencies: + glib-2.0-0.dll + Image has the following delay load dependencies: + optional.dll + Summary + """ + parsed = diagnostic.parse_pe(text) + self.assertEqual(parsed["image_kind"], "exe") + self.assertEqual(parsed["imports"], ["glib-2.0-0.dll"]) + self.assertEqual(parsed["delay_imports"], ["optional.dll"]) + self.assertEqual( + diagnostic.parse_pe(text.replace("EXECUTABLE IMAGE", "DLL"))["image_kind"], + "dll", + ) + with self.assertRaises(ValueError): + diagnostic.parse_pe("truncated") + + def test_manifest_only_and_traversal(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + physical = root / "file with spaces.exe" + physical.write_bytes(b"PE") + manifest = root / "MANIFEST" + manifest.write_text(f"_main/helper.exe {physical}\n") + env = {"RUNFILES_MANIFEST_FILE": str(manifest)} + self.assertEqual( + diagnostic.resolve("_main/helper.exe", env), physical.resolve() + ) + for key in ["../file", "/absolute", "C:/file", "_main/../file"]: + with self.assertRaises(ValueError): + diagnostic.resolve(key, env) + with self.assertRaises(ValueError): + diagnostic.resolve("_main/missing", env) + + def test_ambiguous_runtime(self): + with self.assertRaises(ValueError): + diagnostic.inventory([Path("a/B.dll"), Path("b/b.DLL")]) + + def test_fixture_matches_installed_client_copy_policy(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "source.exe" + source.write_bytes(b"same helper") + runtime = root / "native" / "bin" + runtime.mkdir(parents=True) + library = runtime / "voice.dll" + library.write_bytes(b"library") + marker = runtime.parent / "runtime.json" + marker.write_text("{}") + installed = diagnostic.fixture(root / "fixture", source, [library, marker]) + self.assertEqual(diagnostic.digest(installed), diagnostic.digest(source)) + self.assertTrue((installed.parent / "voice.dll").exists()) + self.assertFalse((installed.parent / "runtime.json").exists()) + self.assertTrue((root / "fixture/bin/codex.exe").exists()) + + def test_file_size_bound(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "large.exe" + path.write_bytes(b"12345") + with patch.object(diagnostic, "MAX_FILE", 4), self.assertRaises(ValueError): + diagnostic.digest(path) + + def test_process_contract_status_and_spawn_error(self): + import io + + child = unittest.mock.Mock(stdout=io.BytesIO(b"commit"), stderr=io.BytesIO(b"")) + child.wait.return_value = 0xC0000135 + env = dict(os.environ) + with patch.object(diagnostic.subprocess, "Popen", return_value=child) as spawn: + result = diagnostic.run( + ["helper.exe", "--build-commit"], env, Path("unrelated") + ) + self.assertIs(spawn.call_args.kwargs["env"], env) + self.assertEqual(spawn.call_args.kwargs["cwd"], Path("unrelated")) + self.assertEqual(result["signed_code"], -1073741515) + self.assertEqual(result["code_u32_hex"], "0xc0000135") + with patch.object( + diagnostic.subprocess, "Popen", side_effect=OSError("missing") + ): + self.assertIn("spawn_error", diagnostic.run(["helper.exe"], env, Path("."))) + + def test_timeout_and_bounded_output(self): + import io + + child = unittest.mock.Mock( + stdout=io.BytesIO(b"x" * 100), stderr=io.BytesIO(b"") + ) + child.wait.side_effect = [ + diagnostic.subprocess.TimeoutExpired("helper", 10), + -1, + ] + with patch.object(diagnostic.subprocess, "Popen", return_value=child): + result = diagnostic.run(["helper"], {}, Path("."), 4) + child.kill.assert_called_once() + self.assertTrue(result["timeout"]) + self.assertTrue(result["truncated"]) + self.assertEqual(result["stdout"], "xxxx") + + def test_diagnose_preserves_source_and_copy_environment(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + files = {} + for name in ["helper.exe", "unit.exe", "dumpbin.exe", "status"]: + path = root / name + path.write_bytes(b"PE") + files[name] = path + files["status"].write_text("STABLE_GIT_COMMIT " + "a" * 40 + "\n") + config = dict( + helper="helper.exe", + unit="unit.exe", + dumpbin="dumpbin.exe", + status="status", + runtime=[], + tool_environment={"TOOL_ONLY": "yes"}, + ) + env = {"TEST_TMPDIR": str(root)} + calls = [] + + def runner(command, environment, cwd, *args): + calls.append((command, dict(environment), cwd)) + text = ( + "a" * 40 + if "--build-commit" in command + else "8664 machine (x64)\nFile Type: EXECUTABLE IMAGE" + ) + return {"code": 0, "stdout": text, "stderr": "", "truncated": False} + + with ( + patch.object( + diagnostic, "resolve", side_effect=lambda key, _: files[key] + ), + patch.object(diagnostic, "run", side_effect=runner), + ): + output = {} + self.assertTrue(diagnostic.diagnose(config, env, output)) + self.assertEqual(calls[0][1:], calls[1][1:]) + self.assertEqual(calls[0][1], env) + self.assertNotIn("TOOL_ONLY", calls[0][1]) + self.assertIn("TOOL_ONLY", calls[-1][1]) + self.assertEqual(output["helper_sha256"], output["installed_sha256"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/third_party/voice/windows_runtime_diagnostic.bzl b/third_party/voice/windows_runtime_diagnostic.bzl new file mode 100644 index 000000000000..a32688bfbe59 --- /dev/null +++ b/third_party/voice/windows_runtime_diagnostic.bzl @@ -0,0 +1,53 @@ +"""Manual, observational Windows loader diagnostic; never packages the binaries.""" + +load("//third_party/voice:windows_native.bzl", "WindowsBuildToolsInfo") + +def _logical(ctx, file): + return file.short_path[3:] if file.short_path.startswith("../") else ctx.workspace_name + "/" + file.short_path + +def _impl(ctx): + tools = ctx.attr._tools[WindowsBuildToolsInfo] + files = tools.files.to_list() + dumpbin = [f for f in files if f.path == tools.inputs["tools"]["dumpbin"]][0] + helper = ctx.executable.helper + unit = ctx.executable.unit + runtime = ctx.attr._runtime[DefaultInfo].default_runfiles + inventory = [f for f in runtime.files.to_list() if f.basename.lower().endswith(".dll") or f.basename == "runtime.json"] + config = ctx.actions.declare_file(ctx.label.name + ".json") + ctx.actions.write(config, json.encode({ + "helper": _logical(ctx, helper), + "unit": _logical(ctx, unit), + "dumpbin": _logical(ctx, dumpbin), + "runtime": [_logical(ctx, f) for f in inventory], + "status": _logical(ctx, ctx.info_file), + "tool_environment": tools.environment, + })) + launcher = ctx.actions.declare_file(ctx.label.name + ".bat") + + # Windows manifest-only runfiles cannot be addressed by concatenating a root. + # Bootstrap just the declared Python/script/config entries via exact keys. + lines = ["@echo off", "setlocal"] + for name, file in [("PYTHON", tools.python.interpreter), ("DRIVER", ctx.file._driver), ("CONFIG", config)]: + key = _logical(ctx, file) + lines.extend([ + 'set "DIAG_%s=%%RUNFILES_DIR%%/%s"' % (name, key), + 'if defined RUNFILES_MANIFEST_FILE for /f "tokens=1,*" %%A in (\'type "%RUNFILES_MANIFEST_FILE%"\') do if "%%A"=="' + key + '" set "DIAG_' + name + '=%%B"', + ]) + lines.extend(['"%DIAG_PYTHON%" "%DIAG_DRIVER%" "%DIAG_CONFIG%"', "exit /b %ERRORLEVEL%"]) + ctx.actions.write(launcher, "\r\n".join(lines) + "\r\n", is_executable = True) + runfiles = ctx.runfiles(files = [config, ctx.file._driver, ctx.info_file, helper, unit], transitive_files = tools.files) + for target in [ctx.attr.helper, ctx.attr.unit, ctx.attr._runtime]: + runfiles = runfiles.merge(target[DefaultInfo].default_runfiles) + return [DefaultInfo(executable = launcher, runfiles = runfiles)] + +windows_runtime_diagnostic_test = rule( + implementation = _impl, + test = True, + attrs = { + "helper": attr.label(mandatory = True, executable = True, cfg = "target"), + "unit": attr.label(mandatory = True, executable = True, cfg = "target"), + "_runtime": attr.label(default = "//third_party/voice:native_link_windows_x86_64"), + "_tools": attr.label(default = "//third_party/voice:windows_tools_x86_64", cfg = "exec", providers = [WindowsBuildToolsInfo]), + "_driver": attr.label(default = "//third_party/voice:windows_runtime_diagnostic.py", allow_single_file = True), + }, +) diff --git a/third_party/voice/windows_runtime_diagnostic.py b/third_party/voice/windows_runtime_diagnostic.py new file mode 100644 index 000000000000..3825e719f5ed --- /dev/null +++ b/third_party/voice/windows_runtime_diagnostic.py @@ -0,0 +1,292 @@ +"""Bounded, manifest-driven Windows PE and unchanged-environment startup evidence.""" + +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import threading +import time + +MAX_FILE = 512 * 1024 * 1024 + + +def resolve(key, env): + if key.startswith(("/", "\\")) or ".." in key.split("/") or ":" in key: + raise ValueError("unsafe runfile key") + manifest = env.get("RUNFILES_MANIFEST_FILE") + if manifest: + with open(manifest, encoding="utf-8") as stream: + for line in stream: + name, _, physical = line.rstrip("\n").partition(" ") + if name == key: + return Path(physical or name).resolve(strict=True) + raise ValueError("missing manifest entry: " + key) + return (Path(env["RUNFILES_DIR"]) / key).resolve(strict=True) + + +def digest(path): + if path.stat().st_size > MAX_FILE: + raise ValueError("PE file size bound exceeded") + value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def run(command, env, cwd, limit=4096): + result = { + "command_kind": Path(command[0]).name, + "executable": command[0], + "cwd": str(cwd), + } + try: + child = subprocess.Popen( + command, env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + except OSError as error: + return dict(result, spawn_error=str(error)) + buffers = [bytearray(), bytearray()] + truncated = [False, False] + + def read(stream, index): + with stream: + while chunk := stream.read(4096): + room = limit - len(buffers[index]) + buffers[index].extend(chunk[:room]) + truncated[index] |= len(chunk) > room + + readers = [ + threading.Thread(target=read, args=(stream, i), daemon=True) + for i, stream in enumerate((child.stdout, child.stderr)) + ] + for reader in readers: + reader.start() + try: + code = child.wait(timeout=10) + except subprocess.TimeoutExpired: + child.kill() + code = child.wait(timeout=5) + result["timeout"] = True + for reader in readers: + reader.join(timeout=1) + result.update( + code=code, + signed_code=code if code < 2**31 else code - 2**32, + code_u32_hex=f"0x{code & 0xFFFFFFFF:08x}", + stdout=buffers[0].decode("utf-8", errors="replace"), + stderr=buffers[1].decode("utf-8", errors="replace"), + truncated=any(truncated), + reader_incomplete=any(r.is_alive() for r in readers), + ) + return result + + +def parse_pe(text): + machine = re.search(r"^\s*([0-9A-Fa-f]+) machine \(([^)]+)\)", text, re.M) + if not machine or not re.search(r"File Type:\s*(DLL|EXECUTABLE IMAGE)", text): + raise ValueError("missing PE machine/image header") + imports, deferred = [], [] + section = None + for line in text.splitlines(): + if "following delay load dependencies" in line.lower(): + section = deferred + elif "following dependencies" in line.lower(): + section = imports + elif line.strip() == "Summary": + section = None + elif section is not None and re.fullmatch(r"\s*[\w.+-]+\.dll\s*", line, re.I): + section.append(line.strip()) + return { + "machine": machine.group(1), + "machine_name": machine.group(2), + "imports": imports, + "delay_imports": deferred, + "image_kind": "dll" if re.search(r"File Type:\s*DLL", text) else "exe", + } + + +def inventory(paths): + result = {} + for path in paths: + name = path.name.lower() + if name in result: + raise ValueError("ambiguous declared runtime basename: " + name) + result[name] = path + return result + + +def fixture(root, helper, runtime): + destination = root / "codex-resources" / "voice" / "bin" + destination.mkdir(parents=True) + (root / "bin").mkdir() + (root / "bin" / "codex.exe").write_bytes(b"") + (root / "codex-package.json").write_text("{}") + copied = destination / helper.name + shutil.copyfile(helper, copied) + for path in runtime: + # installed_client copies every file in native_link/bin, not runtime.json. + if path.parent.name == "bin": + shutil.copyfile(path, destination / path.name) + if digest(helper) != digest(copied): + raise ValueError("installed helper hash mismatch") + return copied + + +def diagnose(config, env, output): + helper, unit, dumpbin, status = [ + resolve(config[key], env) for key in ("helper", "unit", "dumpbin", "status") + ] + runtime = [resolve(key, env) for key in config["runtime"]] + declared = inventory(runtime) + commit = re.findall(r"^STABLE_GIT_COMMIT ([0-9a-f]{40})$", status.read_text(), re.M) + if len(commit) != 1: + raise ValueError("actual workspace status lacks unique build commit") + output.update( + expected_build_commit=commit[0], + dumpbin_sha256=digest(dumpbin), + declared_runtime={ + name: {"path": str(path), "sha256": digest(path)} + for name, path in declared.items() + }, + ) + unrelated = Path(env["TEST_TMPDIR"]) / "unrelated-cwd" + unrelated.mkdir() + copied = fixture(Path(env["TEST_TMPDIR"]) / "installed-fixture", helper, runtime) + output["probes"] = [ + run([str(path), arg], env, unrelated) + for path, arg in ( + (helper, "--build-commit"), + (copied, "--build-commit"), + (unit, "--list"), + ) + ] + output["environment_identical"] = True + output["helper_sha256"] = digest(helper) + output["installed_sha256"] = digest(copied) + output["unit_sha256"] = digest(unit) + output["helper_path"] = str(helper) + adjacent = [ + p + for p in helper.parent.iterdir() + if p.is_file() and p.suffix.lower() in (".dll", ".exe") + ] + if len(adjacent) > 128: + raise ValueError("adjacency inventory bound exceeded") + output["adjacent"] = { + p.name: {"sha256": digest(p), "declared": p.name.lower() in declared} + for p in adjacent + } + tool_env = dict(env, **config["tool_environment"]) + deadline = time.monotonic() + 100 + # Plugins are loaded dynamically; inspect their declared imports too. + queue = [helper, unit] + [p for p in runtime if p.suffix.lower() == ".dll"] + seen, total = set(), 0 + output["pe"] = [] + while queue: + path = queue.pop(0) + if path in seen: + continue + if len(seen) >= 128 or total >= 1024 * 1024 or time.monotonic() >= deadline: + raise ValueError("PE traversal/output bound exceeded") + seen.add(path) + digest(path) + result = run( + [str(dumpbin), "/nologo", "/headers", "/dependents", str(path)], + tool_env, + unrelated, + min(32768, (1024 * 1024 - total) // 2), + ) + total += len(result.get("stdout", "").encode()) + len( + result.get("stderr", "").encode() + ) + if ( + result.get("code") != 0 + or result.get("truncated") + or result.get("timeout") + or result.get("reader_incomplete") + ): + output["pe"].append({"path": str(path), "inspection": result}) + raise ValueError("incomplete dumpbin inspection") + parsed = parse_pe(result["stdout"]) + edges = [] + for name in parsed["imports"] + parsed["delay_imports"]: + dependency = declared.get(name.lower()) + edges.append( + { + "name": name, + "declared": dependency is not None, + "classification": "declared" + if dependency + else "unresolved; system/API-set status unverified", + } + ) + if dependency: + queue.append(dependency) + output["pe"].append( + dict(parsed, path=str(path), edges=edges, inspection=result) + ) + return all( + p.get("code") == 0 + and not p.get("timeout") + and not p.get("truncated") + and not p.get("reader_incomplete") + for p in output["probes"] + ) and all(p["stdout"].strip() == commit[0] for p in output["probes"][:2]) + + +def main(): + env = dict(os.environ) + output = { + "checkout_head": env.get("VOICE_DIAGNOSTIC_HEAD"), + "checkout_tree": env.get("VOICE_DIAGNOSTIC_TREE"), + "schema": 1, + "github": { + key: env.get(key) + for key in ( + "GITHUB_SHA", + "GITHUB_EVENT_NAME", + "GITHUB_REF", + "GITHUB_RUN_ID", + ) + }, + } + success = False + try: + if not all( + re.fullmatch(r"[0-9a-f]{40}", output.get(key) or "") + for key in ("checkout_head", "checkout_tree") + ): + raise ValueError("actual checkout HEAD/tree receipt unavailable") + if output["checkout_head"] != env.get("GITHUB_SHA"): + raise ValueError("checkout HEAD differs from dispatched GitHub SHA") + success = diagnose(json.loads(Path(sys.argv[1]).read_text()), env, output) + except Exception as error: + output["incomplete"] = str(error) + output["success"] = success + destination = Path(env["TEST_UNDECLARED_OUTPUTS_DIR"]) + destination.mkdir(parents=True, exist_ok=True) + (destination / "voice-runtime-diagnostic.json").write_text( + json.dumps(output, indent=2), encoding="utf-8" + ) + print( + json.dumps( + { + "success": success, + "incomplete": output.get("incomplete"), + "probe_statuses": [ + p.get("code_u32_hex", "spawn error") + for p in output.get("probes", []) + ], + } + ) + ) + return 0 if success else 1 + + +if __name__ == "__main__": + sys.exit(main()) From d76a81378e2efe9208f7379415d5892027258700 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 01:47:30 -0400 Subject: [PATCH 02/41] Export Windows voice diagnostic driver for Bazel analysis (Refs #62) --- third_party/voice/BUILD.bazel | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel index 94adba1b5e3b..31ef330ef3a9 100644 --- a/third_party/voice/BUILD.bazel +++ b/third_party/voice/BUILD.bazel @@ -7,7 +7,10 @@ load(":runtime.bzl", "native_runtime") load(":windows_native.bzl", "windows_build_tools", "windows_native_prefix") # All native build consumers must receive the same pinned source manifest. -exports_files(["opus-toolchain.cmake"]) +exports_files([ + "opus-toolchain.cmake", + "windows_runtime_diagnostic.py", +]) filegroup( name = "source_inputs", From 283f462a97ca69a62c7721a84d284239fa8f5a6e Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 02:02:03 -0400 Subject: [PATCH 03/41] Refs #71: bound hosted storage observations and local cache defaults --- .github/actions/prepare-bazel-ci/action.yml | 12 + .github/scripts/hosted_storage.py | 240 +++++++++++++++++ .github/scripts/run-bazel-ci.sh | 76 ++++-- .github/scripts/test_hosted_storage.py | 280 ++++++++++++++++++++ 4 files changed, 584 insertions(+), 24 deletions(-) create mode 100644 .github/scripts/hosted_storage.py create mode 100644 .github/scripts/test_hosted_storage.py diff --git a/.github/actions/prepare-bazel-ci/action.yml b/.github/actions/prepare-bazel-ci/action.yml index 1a98e3bea483..8892b6674e07 100644 --- a/.github/actions/prepare-bazel-ci/action.yml +++ b/.github/actions/prepare-bazel-ci/action.yml @@ -52,6 +52,12 @@ runs: # do not need to be re-downloaded on every CI run. Keep restore failures # non-fatal so transient cache-service errors degrade to a cold build # instead of failing the job. + - name: Observe storage before repository restore + if: runner.environment == 'github-hosted' && (runner.os == 'Linux' || runner.os == 'macOS') && github.repository == 'CompleteDotTech/codex-subagents-multi-provider' + continue-on-error: true + shell: bash + run: python3 .github/scripts/hosted_storage.py observe before-restore none + - name: Restore bazel repository cache id: cache_bazel_repository_restore continue-on-error: true @@ -62,6 +68,12 @@ runs: restore-keys: | ${{ steps.cache_bazel_repository_key.outputs.repository-cache-restore-key }} + - name: Observe storage after repository restore + if: always() && !cancelled() && runner.environment == 'github-hosted' && (runner.os == 'Linux' || runner.os == 'macOS') && github.repository == 'CompleteDotTech/codex-subagents-multi-provider' + continue-on-error: true + shell: bash + run: python3 .github/scripts/hosted_storage.py observe after-restore none + - name: Set up Bazel execution logs shell: bash run: | diff --git a/.github/scripts/hosted_storage.py b/.github/scripts/hosted_storage.py new file mode 100644 index 000000000000..2737450f39bb --- /dev/null +++ b/.github/scripts/hosted_storage.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Scoped disposable-runner storage defaults and bounded numeric receipts.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid + +FORK = "CompleteDotTech/codex-subagents-multi-provider" +INFO_OPTIONS = { + "--config", + "--platforms", + "--host_platform", + "--compilation_mode", + "-c", + "--disk_cache", + "--repository_cache", + "--repo_contents_cache", + "--inject_repository", + "--override_repository", +} +REMOTE_OPTIONS = {"--remote_executor", "--remote_cache", "--strategy"} + + +def scoped(env): + return ( + env.get("GITHUB_ACTIONS") == "true" + and env.get("GITHUB_REPOSITORY") == FORK + and env.get("RUNNER_ENVIRONMENT") == "github-hosted" + and env.get("RUNNER_OS") in ("Linux", "macOS") + and not env.get("BUILDBUDDY_API_KEY") + ) + + +def options(args, names=INFO_OPTIONS): + """Read option/value pairs only before payload; retain repeats and empties.""" + index = 1 # the Bazel command + while index < len(args) and args[index] != "--": + arg = args[index] + name, equal, value = arg.partition("=") + if name in names: + if not equal and index + 1 < len(args) and args[index + 1] != "--": + index += 1 + value = args[index] + yield name, value + index += 1 + + +def defaults(args, env): + explicit = list(options(args, INFO_OPTIONS | REMOTE_OPTIONS)) + if ( + scoped(env) + and args + and args[0] in ("build", "test", "coverage", "run", "info") + and not any(name == "--disk_cache" for name, _ in explicit) + and not any( + (name in REMOTE_OPTIONS and value) + or ( + name == "--config" + and ( + "buildbuddy" in value + or value in ("remote", "ci-linux", "ci-macos", "ci-v8") + ) + ) + for name, value in explicit + ) + ): + return ["--disk_cache="] + return [] + + +def info_options(args): + return [ + name + "=" + value if name != "-c" else "--compilation_mode=" + value + for name, value in options(args) + ] + + +def roots(args, env): + selected = dict(options(args)) + disk = selected.get("--disk_cache", str(Path.home() / ".cache/bazel-disk-cache")) + return { + "output_base": env.get("BAZEL_OUTPUT_BASE"), + "repository_cache": selected.get( + "--repository_cache", env.get("BAZEL_REPOSITORY_CACHE") + ), + "repo_contents_cache": selected.get( + "--repo_contents_cache", env.get("BAZEL_REPO_CONTENTS_CACHE") + ), + "disk_cache": disk, + "ci_temp": env.get("TMP"), + "runner_temp": env.get("RUNNER_TEMP"), + "workspace_archive": env.get("GITHUB_WORKSPACE"), + } + + +def measure(path): + """No traversal listings, symlink following, environment or process dumps.""" + result = {"exists": path.exists()} + ancestor = path + while not ancestor.exists() and ancestor != ancestor.parent: + ancestor = ancestor.parent + stat = ancestor.stat() + fs = os.statvfs(ancestor) + result.update( + device=stat.st_dev, + total_bytes=fs.f_blocks * fs.f_frsize, + free_bytes=fs.f_bfree * fs.f_frsize, + available_bytes=fs.f_bavail * fs.f_frsize, + available_inodes=fs.f_favail, + ) + if result["exists"]: + try: + # -s emits one total, -k is supported by GNU and BSD du; no -L. + run = subprocess.run( + ["du", "-sk", str(path)], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=2, + check=False, + ) + if run.returncode == 0 and len(run.stdout) <= 4096: + result["allocated_bytes"] = int(run.stdout.split(None, 1)[0]) * 1024 + else: + result["allocation_unavailable"] = True + except (OSError, ValueError, subprocess.TimeoutExpired): + result["allocation_unavailable"] = True + return result + + +def receipt(phase, args, env, status=None): + output = { + "schema": 1, + "phase": phase, + "time_unix": time.time(), + "command": args[0] + if args and args[0] in ("build", "test", "coverage", "run") + else "setup", + "exit_code": status, + "runner_os": env.get("RUNNER_OS"), + "run_id": env.get("GITHUB_RUN_ID"), + "attempt": env.get("GITHUB_RUN_ATTEMPT"), + "job": env.get("GITHUB_JOB"), + "paths_are": "declared; rc/config overrides unverified", + "cpu_count": os.cpu_count(), + "roots": {}, + } + seen = {} + for role, raw in roots(args, env).items(): + if not raw: + output["roots"][role] = {"disabled_or_unavailable": True} + continue + try: + path = Path(raw).expanduser().resolve() + entry = {"path": str(path)} + if path in seen: + entry["alias_of"] = seen[path] + else: + entry.update(measure(path)) + entry["overlaps"] = [ + other + for prior, other in seen.items() + if path in prior.parents or prior in path.parents + ] + seen[path] = role + output["roots"][role] = entry + except (OSError, ValueError): + output["roots"][role] = {"unavailable": True} + # Only commit identities are captured, never git configuration or env values. + for key, expression in (("head", "HEAD"), ("tree", "HEAD^{tree}")): + try: + value = ( + subprocess.run( + ["git", "rev-parse", expression], + capture_output=True, + timeout=2, + check=False, + ) + .stdout.decode() + .strip() + ) + if len(value) == 40 and all(c in "0123456789abcdef" for c in value): + output[key] = value + except (OSError, UnicodeError, subprocess.TimeoutExpired): + pass + return output + + +def main(): + mode, *args = sys.argv[1:] + env = dict(os.environ) + if mode in ("defaults", "info"): + values = defaults(args, env) if mode == "defaults" else info_options(args) + for value in values: + sys.stdout.buffer.write(value.encode() + b"\0") + return + if not scoped(env): + return + try: + phase, status, *args = args + if phase not in ( + "before-build", + "after-build", + "before-restore", + "after-restore", + ): + raise ValueError("unknown phase") + output = receipt(phase, args, env, None if status == "none" else int(status)) + text = json.dumps(output, sort_keys=True) + if len(text.encode()) > 32768: + raise ValueError("receipt bound exceeded") + # Print first: a disk-full write can prevent saving the JSON receipt. + print( + "Hosted storage: " + + json.dumps( + { + "phase": phase, + "exit_code": output["exit_code"], + "available_bytes": { + k: v.get("available_bytes") for k, v in output["roots"].items() + }, + } + ), + flush=True, + ) + destination = Path(env["RUNNER_TEMP"]) / "hosted-storage" + destination.mkdir(exist_ok=True) + (destination / (phase + "-" + uuid.uuid4().hex + ".json")).write_text( + text, encoding="utf-8" + ) + except Exception: + # Observation failure must never replace the original build outcome. + print("Hosted storage observation unavailable", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index 976376cc3917..09cda4c96416 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -7,6 +7,11 @@ print_failed_bazel_action_summary=0 remote_download_toplevel=0 windows_msvc_host_platform=0 windows_cross_compile=0 +hosted_storage=0 +if [[ "${GITHUB_ACTIONS:-}" == true && "${GITHUB_REPOSITORY:-}" == CompleteDotTech/codex-subagents-multi-provider && "${RUNNER_ENVIRONMENT:-}" == github-hosted && ( "${RUNNER_OS:-}" == Linux || "${RUNNER_OS:-}" == macOS ) && -z "${BUILDBUDDY_API_KEY:-}" ]]; then + hosted_storage=1 +fi +storage_script="$(dirname "${BASH_SOURCE[0]}")/hosted_storage.py" while [[ $# -gt 0 ]]; do case "$1" in @@ -88,32 +93,40 @@ print_bazel_test_log_tails() { local testlogs_dir local -a bazel_info_args=(info) - if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then - # `bazel info` needs the same CI config as the failed test invocation so - # platform-specific output roots match. On Windows, omitting `ci-windows` - # would point at `local_windows-fastbuild` even when the test ran with the - # MSVC host platform under `local_windows_msvc-fastbuild`. - bazel_info_args+=("--config=${ci_config}") - fi - if [[ $windows_native_fallback -eq 1 ]]; then - if [[ -n "$requested_target_platform" ]]; then - bazel_info_args+=("--platforms=${requested_target_platform}") + if [[ $hosted_storage -eq 1 ]]; then + # Project the same final invocation, preserving split/repeated caller flags. + # Do not pass test/execution-log flags or positional payload to `info`. + while IFS= read -r -d '' arg; do + bazel_info_args+=("$arg") + done < <(python3 "$storage_script" info "${bazel_run_args[@]}") + else + if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then + # `bazel info` needs the same CI config as the failed test invocation so + # platform-specific output roots match. On Windows, omitting `ci-windows` + # would point at `local_windows-fastbuild` even when the test ran with the + # MSVC host platform under `local_windows_msvc-fastbuild`. + bazel_info_args+=("--config=${ci_config}") fi - if [[ -n "$requested_host_platform" ]]; then - bazel_info_args+=("--host_platform=${requested_host_platform}") + if [[ $windows_native_fallback -eq 1 ]]; then + if [[ -n "$requested_target_platform" ]]; then + bazel_info_args+=("--platforms=${requested_target_platform}") + fi + if [[ -n "$requested_host_platform" ]]; then + bazel_info_args+=("--host_platform=${requested_host_platform}") + fi fi - fi - # Only pass flags that affect Bazel's output-root selection or repository - # lookup. Test/build-only flags such as execution logs or remote download - # mode can make `bazel info` fail, which would hide the real test log path. - for arg in "${post_config_bazel_args[@]}"; do - case "$arg" in - --host_platform=* | --platforms=* | --repo_contents_cache=* | --repository_cache=*) - bazel_info_args+=("$arg") - ;; - esac - done + # Only pass flags that affect Bazel's output-root selection or repository + # lookup. Test/build-only flags such as execution logs or remote download + # mode can make `bazel info` fail, which would hide the real test log path. + for arg in "${post_config_bazel_args[@]}"; do + case "$arg" in + --host_platform=* | --platforms=* | --repo_contents_cache=* | --repository_cache=*) + bazel_info_args+=("$arg") + ;; + esac + done + fi testlogs_dir="$(run_bazel_with_startup_args \ --noexperimental_remote_repo_contents_cache \ @@ -539,8 +552,15 @@ bazel_console_log="$(mktemp)" trap 'rm -f "$bazel_console_log"' EXIT bazel_run_args=( - "${bazel_args[@]}" + "${bazel_args[0]}" ) +if [[ $hosted_storage -eq 1 ]]; then + # Early default only: explicit caller flags (including empty/split) win. + while IFS= read -r -d '' arg; do + bazel_run_args+=("$arg") + done < <(python3 "$storage_script" defaults "${bazel_args[@]}") +fi +bazel_run_args+=("${bazel_args[@]:1}") if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then echo "BuildBuddy API key is available; using remote Bazel configuration." bazel_run_args+=("--config=${ci_config}") @@ -550,6 +570,9 @@ fi if (( ${#post_config_bazel_args[@]} > 0 )); then bazel_run_args+=("${post_config_bazel_args[@]}") fi +if [[ $hosted_storage -eq 1 ]]; then + python3 "$storage_script" observe before-build none "${bazel_run_args[@]}" || echo "Hosted storage observation unavailable" >&2 +fi set +e # Work around Bazel 9 remote repo contents cache / overlay materialization # failures seen in CI (for example "is not a symlink" or permission errors @@ -564,6 +587,11 @@ run_bazel_with_startup_args \ bazel_status=${PIPESTATUS[0]} set -e +# Capture filesystem state before any follow-up Bazel information request. +if [[ $hosted_storage -eq 1 ]]; then + python3 "$storage_script" observe after-build "$bazel_status" "${bazel_run_args[@]}" || echo "Hosted storage observation unavailable" >&2 +fi + if [[ ${bazel_status:-0} -ne 0 ]]; then # Observe the failed build outputs before any diagnostic Bazel info call. if [[ "${RUNNER_OS:-}" == Windows && "${GITHUB_ACTIONS:-}" == true && "${GITHUB_REPOSITORY:-}" == CompleteDotTech/codex-subagents-multi-provider && -n "${CODEX_BAZEL_EXECUTION_LOG_COMPACT_DIR:-}" ]]; then diff --git a/.github/scripts/test_hosted_storage.py b/.github/scripts/test_hosted_storage.py new file mode 100644 index 000000000000..91fd15e59a16 --- /dev/null +++ b/.github/scripts/test_hosted_storage.py @@ -0,0 +1,280 @@ +"""No-building tests for storage policy, bounded observations and shell wiring.""" + +import contextlib +import io +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import hosted_storage as storage + +ENV = dict( + GITHUB_ACTIONS="true", + GITHUB_REPOSITORY=storage.FORK, + RUNNER_ENVIRONMENT="github-hosted", + RUNNER_OS="Linux", +) + + +class PolicyTests(unittest.TestCase): + def test_default_is_scoped_and_keeps_remote_configs_out(self): + for os_name in ("Linux", "macOS"): + self.assertEqual( + storage.defaults(["test"], dict(ENV, RUNNER_OS=os_name)), + ["--disk_cache="], + ) + for change in ( + {"GITHUB_ACTIONS": "false"}, + {"GITHUB_REPOSITORY": "openai/codex"}, + {"RUNNER_ENVIRONMENT": "self-hosted"}, + {"RUNNER_OS": "Windows"}, + {"BUILDBUDDY_API_KEY": "synthetic"}, + ): + self.assertEqual(storage.defaults(["test"], dict(ENV, **change)), []) + self.assertEqual(storage.defaults(["clean"], ENV), []) + + def test_explicit_empty_split_and_repeated_settings_and_payload(self): + for flags in ( + ["--disk_cache="], + ["--disk_cache", ""], + ["--disk_cache", "cache with spaces"], + ["--disk_cache=first", "--disk_cache", "last"], + ): + self.assertEqual(storage.defaults(["test", *flags], ENV), []) + args = ["test", "--", "//target", "--disk_cache=payload"] + self.assertEqual(storage.defaults(args, ENV), ["--disk_cache="]) + self.assertEqual(storage.info_options(args), []) + + def test_explicit_remote_routes_are_not_given_local_defaults(self): + for flags in ( + ["--remote_executor=grpcs://synthetic"], + ["--remote_cache", "https://synthetic"], + ["--config", "buildbuddy-generic-rbe"], + ["--config=ci-linux"], + ["--strategy=remote"], + ): + self.assertEqual(storage.defaults(["test", *flags], ENV), []) + + def test_info_projection_preserves_order_and_actual_configuration(self): + args = [ + "test", + "--disk_cache=", + "--compilation_mode", + "opt", + "-c", + "fastbuild", + "--platforms=//:one", + "--host_platform", + "//:host", + "--config", + "caller", + "--repository_cache", + "a b", + "--execution_log_compact_file=omit", + "--test_env=SECRET=omit", + "--", + "--platforms=payload", + ] + self.assertEqual( + storage.info_options(args), + [ + "--disk_cache=", + "--compilation_mode=opt", + "--compilation_mode=fastbuild", + "--platforms=//:one", + "--host_platform=//:host", + "--config=caller", + "--repository_cache=a b", + ], + ) + + +class ObservationTests(unittest.TestCase): + def test_numeric_capacity_and_bounded_du(self): + fs = SimpleNamespace( + f_blocks=100, f_bfree=30, f_bavail=20, f_frsize=4096, f_favail=12 + ) + with ( + tempfile.TemporaryDirectory() as directory, + patch.object(storage.os, "statvfs", return_value=fs, create=True), + patch.object( + storage.subprocess, + "run", + return_value=SimpleNamespace(returncode=0, stdout=b"4\tignored\n"), + ) as run, + ): + result = storage.measure(Path(directory)) + self.assertEqual(result["available_bytes"], 81920) + self.assertEqual(result["allocated_bytes"], 4096) + self.assertEqual(run.call_args.kwargs["timeout"], 2) + self.assertNotIn("-L", run.call_args.args[0]) + + def test_du_timeout_is_explicit_not_zero(self): + fs = SimpleNamespace( + f_blocks=100, f_bfree=30, f_bavail=20, f_frsize=4096, f_favail=12 + ) + with ( + tempfile.TemporaryDirectory() as directory, + patch.object(storage.os, "statvfs", return_value=fs, create=True), + patch.object( + storage.subprocess, + "run", + side_effect=subprocess.TimeoutExpired("du", 2), + ), + ): + result = storage.measure(Path(directory)) + self.assertTrue(result["allocation_unavailable"]) + self.assertNotIn("allocated_bytes", result) + + def test_aliases_overlaps_missing_and_no_environment_dump(self): + with tempfile.TemporaryDirectory() as directory: + env = dict( + ENV, + BAZEL_OUTPUT_BASE=directory, + BAZEL_REPOSITORY_CACHE=directory, + TMP=str(Path(directory) / "tmp"), + SECRET="must-not-appear", + ) + with ( + patch.object(storage, "measure", return_value={"available_bytes": 42}), + patch.object(storage.subprocess, "run", side_effect=OSError), + ): + result = storage.receipt( + "after-build", ["test", "--disk_cache="], env, 37 + ) + self.assertEqual(result["exit_code"], 37) + self.assertEqual(result["roots"]["repository_cache"]["alias_of"], "output_base") + self.assertIn("output_base", result["roots"]["ci_temp"]["overlaps"]) + self.assertTrue(result["roots"]["disk_cache"]["disabled_or_unavailable"]) + self.assertNotIn("must-not-appear", json.dumps(result)) + + def test_storage_errors_never_replace_build_outcome_or_expose_error_text(self): + out = io.StringIO() + with ( + patch.dict(os.environ, ENV, clear=True), + patch.object( + sys, "argv", ["script", "observe", "after-build", "37", "test"] + ), + patch.object(storage, "receipt", side_effect=OSError("SECRET")), + contextlib.redirect_stderr(out), + ): + storage.main() + self.assertEqual(out.getvalue(), "Hosted storage observation unavailable\n") + + def test_numeric_summary_survives_receipt_disk_write_failure(self): + out, errors = io.StringIO(), io.StringIO() + result = {"exit_code": 37, "roots": {"output_base": {"available_bytes": 0}}} + with ( + patch.dict(os.environ, dict(ENV, RUNNER_TEMP="unused"), clear=True), + patch.object( + sys, "argv", ["script", "observe", "after-build", "37", "test"] + ), + patch.object(storage, "receipt", return_value=result), + patch.object(Path, "mkdir", side_effect=OSError("disk full")), + contextlib.redirect_stdout(out), + contextlib.redirect_stderr(errors), + ): + storage.main() + self.assertIn('"output_base": 0', out.getvalue()) + self.assertIn('"exit_code": 37', out.getvalue()) + self.assertIn("observation unavailable", errors.getvalue()) + + +class ShellTests(unittest.TestCase): + def invoke(self, flags=(), extra=()): + bash = ( + str( + Path(os.environ.get("ProgramFiles", "C:/Program Files")) + / "Git/bin/bash.exe" + ) + if os.name == "nt" + else shutil.which("bash") + ) + if not bash: + self.skipTest("bash unavailable") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = Path(__file__).parent + shutil.copyfile(source / "run-bazel-ci.sh", root / "run-bazel-ci.sh") + shutil.copyfile(source / "hosted_storage.py", root / "policy.py") + # Mock just filesystem observations; use real defaults/projection. + (root / "hosted_storage.py").write_text( + "import json, os, sys\nimport policy\n" + "if sys.argv[1] == 'observe':\n" + " with open(os.environ['RECORD'], 'a') as f: f.write(json.dumps(sys.argv[2:4])+'\\n')\n" + "else: policy.main()\n", + encoding="utf-8", + ) + (root / "run_bazel_with_buildbuddy.py").write_text( + "#!/usr/bin/env python3\nimport json, os, sys\n" + "with open(os.environ['RECORD'], 'a') as f: f.write(json.dumps(sys.argv[1:])+'\\n')\n" + "if 'info' in sys.argv: print('/nonexistent-testlogs'); sys.exit(0)\n" + "sys.exit(37)\n", + encoding="utf-8", + ) + (root / "run_bazel_with_buildbuddy.py").chmod(0o755) + record = root / "record" + env = { + key: value + for key, value in os.environ.items() + if key in ("PATH", "SYSTEMROOT", "SystemRoot", "TEMP", "TMP") + } + env.update(ENV, RECORD=str(record), MSYS2_ARG_CONV_EXCL="*") + result = subprocess.run( + [ + bash, + str(root / "run-bazel-ci.sh"), + "--print-failed-test-logs", + "--", + "test", + *flags, + "--", + "//target", + *extra, + ], + cwd=root, + env=env, + capture_output=True, + text=True, + timeout=20, + ) + return result, [ + json.loads(line) for line in record.read_text().splitlines() + ] + + def test_failure_snapshot_precedes_info_and_original_exit_survives(self): + result, calls = self.invoke( + ["--compilation_mode", "opt", "--platforms", "//:custom"] + ) + self.assertEqual(result.returncode, 37, result.stderr) + self.assertEqual(calls[0], ["before-build", "none"]) + self.assertEqual(calls[2], ["after-build", "37"]) + self.assertIn("info", calls[3]) + self.assertIn("--disk_cache=", calls[1]) + self.assertIn("--disk_cache=", calls[3]) + self.assertIn("--compilation_mode=opt", calls[3]) + self.assertIn("--platforms=//:custom", calls[3]) + + def test_explicit_split_cache_and_payload_are_not_changed(self): + result, calls = self.invoke( + ["--disk_cache", "cache with spaces"], ["--disk_cache=payload"] + ) + self.assertEqual(result.returncode, 37, result.stderr) + build = calls[1] + self.assertNotIn("--disk_cache=", build) + self.assertEqual( + build[build.index("--") + 1 :], ["//target", "--disk_cache=payload"] + ) + self.assertIn("--disk_cache=cache with spaces", calls[3]) + self.assertNotIn("--disk_cache=payload", calls[3]) + + +if __name__ == "__main__": + unittest.main() From 0f13f16448d7e62b6f826aeb8a015568ee7853af Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 02:13:16 -0400 Subject: [PATCH 04/41] Refs #71: normalize keyless configs before storage policy --- .github/scripts/hosted_storage.py | 67 +++++++++++--- .github/scripts/run-bazel-ci.sh | 20 +++-- .github/scripts/test_hosted_storage.py | 120 +++++++++++++++++++++++-- 3 files changed, 184 insertions(+), 23 deletions(-) diff --git a/.github/scripts/hosted_storage.py b/.github/scripts/hosted_storage.py index 2737450f39bb..b5e9ceb003c5 100644 --- a/.github/scripts/hosted_storage.py +++ b/.github/scripts/hosted_storage.py @@ -9,6 +9,11 @@ import time import uuid +from run_bazel_with_buildbuddy import ( + REMOTE_EXECUTION_CONFIGS, + bazel_args_without_remote_execution, +) + FORK = "CompleteDotTech/codex-subagents-multi-provider" INFO_OPTIONS = { "--config", @@ -50,28 +55,65 @@ def options(args, names=INFO_OPTIONS): def defaults(args, env): + args = local_arguments(args, env) explicit = list(options(args, INFO_OPTIONS | REMOTE_OPTIONS)) + # --strategy is a per-mnemonic map: later values replace the same mnemonic. + # A local/sandboxed override does not select a remote execution strategy. + strategies = {} + for name, value in explicit: + mnemonic, equal, choices = value.partition("=") + if name == "--strategy" and equal: + strategies[mnemonic] = choices.split(",") if ( scoped(env) and args and args[0] in ("build", "test", "coverage", "run", "info") and not any(name == "--disk_cache" for name, _ in explicit) and not any( - (name in REMOTE_OPTIONS and value) - or ( - name == "--config" - and ( - "buildbuddy" in value - or value in ("remote", "ci-linux", "ci-macos", "ci-v8") - ) - ) + (name in ("--remote_executor", "--remote_cache") and value) + or (name == "--config" and ("buildbuddy" in value or value == "remote")) for name, value in explicit ) + and not any("remote" in choices for choices in strategies.values()) ): return ["--disk_cache="] return [] +def local_arguments(args, env): + """Match the existing keyless backend, including known split CI configs. + + Other routes retain their original bytes. Only configs already classified by + the backend as credential-dependent remote CI are canonicalized and removed. + """ + if not scoped(env): + return list(args) + normalized = [] + index = 0 + while index < len(args): + if args[index] == "--": + normalized.extend(args[index:]) + break + if ( + args[index] == "--config" + and index + 1 < len(args) + and "--config=" + args[index + 1] in REMOTE_EXECUTION_CONFIGS + ): + normalized.append("--config=" + args[index + 1]) + index += 2 + else: + normalized.append(args[index]) + index += 1 + return bazel_args_without_remote_execution(normalized) + + +def effective_arguments(args, env): + args = local_arguments(args, env) + if not args: + return args + return [args[0], *defaults(args, env), *args[1:]] + + def info_options(args): return [ name + "=" + value if name != "-c" else "--compilation_mode=" + value @@ -192,8 +234,13 @@ def receipt(phase, args, env, status=None): def main(): mode, *args = sys.argv[1:] env = dict(os.environ) - if mode in ("defaults", "info"): - values = defaults(args, env) if mode == "defaults" else info_options(args) + if mode in ("defaults", "effective", "info"): + if mode == "defaults": + values = defaults(args, env) + elif mode == "effective": + values = effective_arguments(args, env) + else: + values = info_options(effective_arguments(args, env)) for value in values: sys.stdout.buffer.write(value.encode() + b"\0") return diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index 09cda4c96416..e1c41fc863c4 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -552,15 +552,8 @@ bazel_console_log="$(mktemp)" trap 'rm -f "$bazel_console_log"' EXIT bazel_run_args=( - "${bazel_args[0]}" + "${bazel_args[@]}" ) -if [[ $hosted_storage -eq 1 ]]; then - # Early default only: explicit caller flags (including empty/split) win. - while IFS= read -r -d '' arg; do - bazel_run_args+=("$arg") - done < <(python3 "$storage_script" defaults "${bazel_args[@]}") -fi -bazel_run_args+=("${bazel_args[@]:1}") if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then echo "BuildBuddy API key is available; using remote Bazel configuration." bazel_run_args+=("--config=${ci_config}") @@ -571,6 +564,17 @@ if (( ${#post_config_bazel_args[@]} > 0 )); then bazel_run_args+=("${post_config_bazel_args[@]}") fi if [[ $hosted_storage -eq 1 ]]; then + # Use the backend's keyless config classification before deciding defaults; + # BUILD and INFO must see the same effective local arguments. + effective_bazel_args=() + while IFS= read -r -d '' arg; do + effective_bazel_args+=("$arg") + done < <(python3 "$storage_script" effective "${bazel_run_args[@]}") + if (( ${#effective_bazel_args[@]} == 0 )); then + echo "Could not normalize hosted Bazel arguments" >&2 + exit 1 + fi + bazel_run_args=("${effective_bazel_args[@]}") python3 "$storage_script" observe before-build none "${bazel_run_args[@]}" || echo "Hosted storage observation unavailable" >&2 fi set +e diff --git a/.github/scripts/test_hosted_storage.py b/.github/scripts/test_hosted_storage.py index 91fd15e59a16..861df99a8f57 100644 --- a/.github/scripts/test_hosted_storage.py +++ b/.github/scripts/test_hosted_storage.py @@ -14,6 +14,7 @@ from unittest.mock import patch import hosted_storage as storage +import run_bazel_with_buildbuddy as backend ENV = dict( GITHUB_ACTIONS="true", @@ -57,11 +58,100 @@ def test_explicit_remote_routes_are_not_given_local_defaults(self): ["--remote_executor=grpcs://synthetic"], ["--remote_cache", "https://synthetic"], ["--config", "buildbuddy-generic-rbe"], - ["--config=ci-linux"], - ["--strategy=remote"], + ["--strategy=Rustc=remote"], ): self.assertEqual(storage.defaults(["test", *flags], ENV), []) + def test_local_only_strategies_and_last_value_for_same_mnemonic(self): + for flags in ( + ["--strategy=Rustc=local"], + ["--strategy", "Rustc=local,sandboxed"], + ["--strategy=Rustc=remote", "--strategy=Rustc=local"], + ): + args = ["test", *flags] + self.assertEqual( + storage.effective_arguments(args, ENV), + ["test", "--disk_cache=", *flags], + ) + for flags in ( + ["--strategy=Rustc=local", "--strategy=Rustc=remote,sandboxed"], + ["--strategy=Rustc=local", "--strategy=Genrule=remote"], + ): + self.assertEqual(storage.defaults(["test", *flags], ENV), []) + + +class BackendInteractionTests(unittest.TestCase): + def test_keyless_configs_have_identical_local_build_and_info_routes(self): + for name in ("ci-linux", "ci-macos", "ci-v8"): + for flags in (["--config=" + name], ["--config", name]): + with self.subTest(flags=flags): + payload = ["//target", "--config=" + name, "--config", name] + effective = storage.effective_arguments( + ["test", *flags, "--", *payload], ENV + ) + build = backend.bazel_command(*effective, env=ENV) + info = backend.bazel_command( + "info", + *storage.info_options(effective), + "bazel-testlogs", + env=ENV, + ) + self.assertIn("--disk_cache=", build) + self.assertIn("--disk_cache=", info) + self.assertNotIn("--config=" + name, build[: build.index("--")]) + self.assertNotIn("--config=" + name, info) + self.assertNotIn(name, build[: build.index("--")]) + self.assertEqual(build[build.index("--") + 1 :], payload) + self.assertEqual( + storage.effective_arguments(effective, ENV), effective + ) + + def test_explicit_cache_remains_owned_after_real_backend_fallback(self): + for cache in ( + ["--disk_cache=first", "--disk_cache", "last"], + ["--disk_cache", ""], + ["--disk_cache="], + ): + effective = storage.effective_arguments( + ["test", "--config", "ci-linux", *cache, "--", "//target"], ENV + ) + command = backend.bazel_command(*effective, env=ENV) + self.assertEqual( + command[command.index("test") + 1 : command.index("--")], cache + ) + + def test_new_normalization_does_not_change_other_routes(self): + for change in ( + {"BUILDBUDDY_API_KEY": "synthetic-key"}, + {"GITHUB_REPOSITORY": "openai/codex"}, + {"RUNNER_OS": "Windows"}, + {"GITHUB_ACTIONS": "false"}, + ): + env = dict(ENV, **change) + for flags in (["--config=ci-linux"], ["--config", "ci-linux"]): + args = ["test", *flags, "--", "//target"] + effective = storage.effective_arguments(args, env) + self.assertEqual(effective, args) + self.assertEqual( + backend.bazel_command(*effective, env=env), + backend.bazel_command(*args, env=env), + ) + + def test_real_remote_endpoints_are_retained_by_actual_backend(self): + for flags in ( + ["--remote_executor=grpcs://synthetic"], + ["--remote_cache", "https://synthetic"], + ): + effective = storage.effective_arguments( + ["test", "--config=ci-linux", *flags, "--", "//target"], ENV + ) + command = backend.bazel_command(*effective, env=ENV) + self.assertNotIn("--disk_cache=", command) + for flag in flags: + self.assertIn(flag, command) + + +class ProjectionTests(unittest.TestCase): def test_info_projection_preserves_order_and_actual_configuration(self): args = [ "test", @@ -204,6 +294,9 @@ def invoke(self, flags=(), extra=()): source = Path(__file__).parent shutil.copyfile(source / "run-bazel-ci.sh", root / "run-bazel-ci.sh") shutil.copyfile(source / "hosted_storage.py", root / "policy.py") + shutil.copyfile( + source / "run_bazel_with_buildbuddy.py", root / "backend.py" + ) # Mock just filesystem observations; use real defaults/projection. (root / "hosted_storage.py").write_text( "import json, os, sys\nimport policy\n" @@ -214,9 +307,11 @@ def invoke(self, flags=(), extra=()): ) (root / "run_bazel_with_buildbuddy.py").write_text( "#!/usr/bin/env python3\nimport json, os, sys\n" - "with open(os.environ['RECORD'], 'a') as f: f.write(json.dumps(sys.argv[1:])+'\\n')\n" - "if 'info' in sys.argv: print('/nonexistent-testlogs'); sys.exit(0)\n" - "sys.exit(37)\n", + "from backend import REMOTE_EXECUTION_CONFIGS, bazel_args_without_remote_execution\n" + "if __name__ == '__main__':\n" + " with open(os.environ['RECORD'], 'a') as f: f.write(json.dumps(sys.argv[1:])+'\\n')\n" + " if 'info' in sys.argv: print('/nonexistent-testlogs'); sys.exit(0)\n" + " sys.exit(37)\n", encoding="utf-8", ) (root / "run_bazel_with_buildbuddy.py").chmod(0o755) @@ -275,6 +370,21 @@ def test_explicit_split_cache_and_payload_are_not_changed(self): self.assertIn("--disk_cache=cache with spaces", calls[3]) self.assertNotIn("--disk_cache=payload", calls[3]) + def test_both_remote_ci_spellings_are_removed_before_shell_dispatch(self): + for name in ("ci-linux", "ci-macos", "ci-v8"): + for flags in (["--config=" + name], ["--config", name]): + result, calls = self.invoke(flags, ["--config=" + name]) + self.assertEqual(result.returncode, 37, result.stderr) + build, info = calls[1], calls[3] + self.assertIn("--disk_cache=", build) + self.assertIn("--disk_cache=", info) + self.assertNotIn("--config=" + name, build[: build.index("--")]) + self.assertNotIn(name, build[: build.index("--")]) + self.assertNotIn("--config=" + name, info) + self.assertEqual( + build[build.index("--") + 1 :], ["//target", "--config=" + name] + ) + if __name__ == "__main__": unittest.main() From 5b5a839655f9144030033ff7ecad987930156e3c Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 02:18:03 -0400 Subject: [PATCH 05/41] Refs #71: honor final scalar remote endpoint settings --- .github/scripts/hosted_storage.py | 10 +++- .github/scripts/test_hosted_storage.py | 72 ++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/.github/scripts/hosted_storage.py b/.github/scripts/hosted_storage.py index b5e9ceb003c5..bc7b3b4e7d35 100644 --- a/.github/scripts/hosted_storage.py +++ b/.github/scripts/hosted_storage.py @@ -57,6 +57,12 @@ def options(args, names=INFO_OPTIONS): def defaults(args, env): args = local_arguments(args, env) explicit = list(options(args, INFO_OPTIONS | REMOTE_OPTIONS)) + # These are scalar options: each endpoint's final value is authoritative. + endpoints = { + name: value + for name, value in explicit + if name in ("--remote_executor", "--remote_cache") + } # --strategy is a per-mnemonic map: later values replace the same mnemonic. # A local/sandboxed override does not select a remote execution strategy. strategies = {} @@ -70,10 +76,10 @@ def defaults(args, env): and args[0] in ("build", "test", "coverage", "run", "info") and not any(name == "--disk_cache" for name, _ in explicit) and not any( - (name in ("--remote_executor", "--remote_cache") and value) - or (name == "--config" and ("buildbuddy" in value or value == "remote")) + name == "--config" and ("buildbuddy" in value or value == "remote") for name, value in explicit ) + and not any(endpoints.values()) and not any("remote" in choices for choices in strategies.values()) ): return ["--disk_cache="] diff --git a/.github/scripts/test_hosted_storage.py b/.github/scripts/test_hosted_storage.py index 861df99a8f57..021c8ab6b2bf 100644 --- a/.github/scripts/test_hosted_storage.py +++ b/.github/scripts/test_hosted_storage.py @@ -81,6 +81,78 @@ def test_local_only_strategies_and_last_value_for_same_mnemonic(self): class BackendInteractionTests(unittest.TestCase): + def test_final_endpoint_values_control_local_default_without_rewriting_flags(self): + cases = [ + (["--remote_cache=https://synthetic", "--remote_cache="], True), + (["--remote_cache=", "--remote_cache", "https://synthetic"], False), + (["--remote_executor", "grpcs://synthetic", "--remote_executor", ""], True), + (["--remote_executor", "", "--remote_executor=grpcs://synthetic"], False), + ( + [ + "--remote_cache=https://synthetic", + "--remote_executor=grpcs://synthetic", + "--remote_cache", + "", + "--remote_executor=", + ], + True, + ), + ( + [ + "--remote_cache=", + "--remote_executor=grpcs://synthetic", + "--remote_cache=", + ], + False, + ), + ( + [ + "--remote_executor=", + "--remote_cache=https://synthetic", + "--remote_executor", + "", + ], + False, + ), + ] + payload = [ + "//target", + "--remote_cache=https://payload", + "--remote_executor", + "payload", + ] + for flags, local in cases: + with self.subTest(flags=flags): + effective = storage.effective_arguments( + ["test", *flags, "--", *payload], ENV + ) + command = backend.bazel_command(*effective, env=ENV) + before = command[command.index("test") + 1 : command.index("--")] + self.assertEqual(before, (["--disk_cache="] if local else []) + flags) + self.assertEqual(command[command.index("--") + 1 :], payload) + + def test_empty_endpoints_do_not_override_other_explicit_remote_or_cache_choices( + self, + ): + endpoints = [ + "--remote_cache=https://synthetic", + "--remote_cache=", + "--remote_executor", + "", + ] + for choice in ( + ["--disk_cache", "owned"], + ["--disk_cache="], + ["--config=buildbuddy-generic-rbe"], + ["--strategy=Rustc=remote"], + ): + effective = storage.effective_arguments( + ["test", *endpoints, *choice, "--", "//target"], ENV + ) + command = backend.bazel_command(*effective, env=ENV) + before = command[command.index("test") + 1 : command.index("--")] + self.assertEqual(before, endpoints + choice) + def test_keyless_configs_have_identical_local_build_and_info_routes(self): for name in ("ci-linux", "ci-macos", "ci-v8"): for flags in (["--config=" + name], ["--config", name]): From 0f828ddd790459c4e21dd339a845a682b08527a3 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 02:22:56 -0400 Subject: [PATCH 06/41] Resolve Windows diagnostic runfiles without nested CMD commands (Refs #62) --- third_party/voice/BUILD.bazel | 1 + ...est_windows_runtime_diagnostic_launcher.py | 173 ++++++++++++++++++ .../voice/windows_runtime_diagnostic.bat.tpl | 76 ++++++++ .../voice/windows_runtime_diagnostic.bzl | 22 +-- .../voice/windows_runtime_diagnostic.py | 1 + 5 files changed, 262 insertions(+), 11 deletions(-) create mode 100644 third_party/voice/test_windows_runtime_diagnostic_launcher.py create mode 100644 third_party/voice/windows_runtime_diagnostic.bat.tpl diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel index 31ef330ef3a9..69db38b19754 100644 --- a/third_party/voice/BUILD.bazel +++ b/third_party/voice/BUILD.bazel @@ -9,6 +9,7 @@ load(":windows_native.bzl", "windows_build_tools", "windows_native_prefix") # All native build consumers must receive the same pinned source manifest. exports_files([ "opus-toolchain.cmake", + "windows_runtime_diagnostic.bat.tpl", "windows_runtime_diagnostic.py", ]) diff --git a/third_party/voice/test_windows_runtime_diagnostic_launcher.py b/third_party/voice/test_windows_runtime_diagnostic_launcher.py new file mode 100644 index 000000000000..0ebf931d065e --- /dev/null +++ b/third_party/voice/test_windows_runtime_diagnostic_launcher.py @@ -0,0 +1,173 @@ +"""Execute the actual rendered launcher with harmless fixtures under Windows CMD.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +@unittest.skipUnless(os.name == "nt", "Windows CMD bootstrap regression") +class LauncherTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="voice launcher spaces ") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.cmd = Path(os.environ["SystemRoot"]) / "System32/cmd.exe" + self.python = Path(sys.executable).resolve(strict=True) + self.keys = { + "__PYTHON__": "_diag_python/" + self.python.name, + "__DRIVER__": "_main/driver.py", + "__CONFIG__": "_main/config.json", + } + self.launcher = self.root / "diagnostic launcher.bat" + template = ( + Path(__file__) + .with_name("windows_runtime_diagnostic.bat.tpl") + .read_bytes() + .decode() + ) + for token, key in self.keys.items(): + template = template.replace(token, key) + self.launcher.write_bytes(template.encode()) + self.output = self.root / "observed.json" + self.driver = self.root / "harmless driver.py" + self.driver.write_text( + "import json,os,sys\n" + "from pathlib import Path\n" + f"sys.path.insert(0, {str(Path(__file__).parent)!r})\n" + "from windows_runtime_diagnostic import resolve\n" + "config=json.loads(Path(sys.argv[1]).read_text())\n" + "assert resolve('_main/config.json',os.environ)==Path(sys.argv[1]).resolve()\n" + "Path(config['output']).write_text(json.dumps({k:os.environ.get(k) for k in " + "['RUNFILES_MANIFEST_FILE','RUNFILES_DIR','TEST_SRCDIR','VOICE_DIAGNOSTIC_RUNFILES_REASON']}))\n" + ) + self.config = self.root / "harmless config.json" + self.config.write_text(json.dumps({"output": str(self.output)})) + self.env = dict(os.environ) + for key in ["RUNFILES_MANIFEST_FILE", "RUNFILES_DIR", "TEST_SRCDIR"]: + self.env.pop(key, None) + + def manifest(self, path, missing=None, nonexistent=None): + physical = { + "__PYTHON__": self.python, + "__DRIVER__": self.driver, + "__CONFIG__": self.config, + } + path.write_text( + "".join( + f"{key} {self.root / 'absent' if token == nonexistent else physical[token]}\n" + for token, key in self.keys.items() + if token != missing + ) + ) + + def run_launcher(self): + result = subprocess.run( + f'"{self.cmd}" /d /s /c ""{self.launcher}""', + cwd=self.root, + env=self.env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=15, + ) + return result + + def assert_success(self, result): + self.assertEqual(result.returncode, 0, result.stdout) + self.assertTrue(self.output.exists(), result.stdout) + return json.loads(self.output.read_text()) + + def test_manifest_only_with_spaces(self): + path = self.root / "manifest with spaces" + self.manifest(path) + self.env["RUNFILES_MANIFEST_FILE"] = str(path) + observed = self.assert_success(self.run_launcher()) + self.assertEqual(observed["RUNFILES_MANIFEST_FILE"], str(path)) + self.assertFalse(observed["RUNFILES_DIR"]) + self.assertEqual( + observed["VOICE_DIAGNOSTIC_RUNFILES_REASON"], + "existing_environment_manifest", + ) + + def test_directory_only(self): + root = self.root / "runfiles directory" + (root / "_main").mkdir(parents=True) + # A directory junction preserves the concrete interpreter's support tree. + junction = root / "_diag_python" + command = ( + f'"{self.cmd}" /d /s /c "mklink /J "{junction}" "{self.python.parent}""' + ) + made = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=10, + ) + self.assertEqual(made.returncode, 0, made.stdout) + self.addCleanup(lambda: junction.rmdir() if junction.exists() else None) + (root / self.keys["__DRIVER__"]).write_bytes(self.driver.read_bytes()) + (root / self.keys["__CONFIG__"]).write_bytes(self.config.read_bytes()) + self.env["TEST_SRCDIR"] = str(root) + self.env["RUNFILES_MANIFEST_FILE"] = str(self.root / "bogus manifest") + observed = self.assert_success(self.run_launcher()) + self.assertFalse(observed["RUNFILES_MANIFEST_FILE"]) + self.assertEqual(observed["RUNFILES_DIR"], str(root)) + self.assertEqual(observed["TEST_SRCDIR"], str(root)) + + def test_absent_or_bogus_environment_adjacent_manifest_fallback(self): + manifest = Path(str(self.launcher) + ".runfiles_manifest") + self.manifest(manifest) + for bogus in [False, True]: + with self.subTest(bogus=bogus): + if bogus: + self.env["RUNFILES_MANIFEST_FILE"] = str( + self.root / "missing manifest" + ) + self.env["RUNFILES_DIR"] = str(self.root / "missing root") + observed = self.assert_success(self.run_launcher()) + self.assertEqual(observed["RUNFILES_MANIFEST_FILE"], str(manifest)) + self.assertEqual( + observed["VOICE_DIAGNOSTIC_RUNFILES_REASON"], + "existing_adjacent_manifest", + ) + + def test_manifest_inside_environment_runfiles_directory(self): + root = self.root / "manifest only runfiles directory" + root.mkdir() + manifest = root / "MANIFEST" + self.manifest(manifest) + self.env["RUNFILES_DIR"] = str(root) + observed = self.assert_success(self.run_launcher()) + self.assertEqual(Path(observed["RUNFILES_MANIFEST_FILE"]), manifest) + self.assertEqual( + observed["VOICE_DIAGNOSTIC_RUNFILES_REASON"], "existing_directory_manifest" + ) + + def test_missing_and_nonexistent_roles_fail_clearly(self): + manifest = self.root / "MANIFEST" + self.env["RUNFILES_MANIFEST_FILE"] = str(manifest) + for token, role in [ + ("__PYTHON__", "python"), + ("__DRIVER__", "driver"), + ("__CONFIG__", "config"), + ]: + for missing_file in [False, True]: + with self.subTest(role=role, missing_file=missing_file): + self.manifest( + manifest, + missing=None if missing_file else token, + nonexistent=token if missing_file else None, + ) + result = self.run_launcher() + self.assertNotEqual(result.returncode, 0) + self.assertIn(role + " runfile", result.stdout) + self.assertFalse(self.output.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/third_party/voice/windows_runtime_diagnostic.bat.tpl b/third_party/voice/windows_runtime_diagnostic.bat.tpl new file mode 100644 index 000000000000..0aa6b215cc98 --- /dev/null +++ b/third_party/voice/windows_runtime_diagnostic.bat.tpl @@ -0,0 +1,76 @@ +@echo off +setlocal EnableExtensions EnableDelayedExpansion + +rem Choose one existing runfiles source for both bootstrap and Python resolution. +set "diag_manifest=" +set "diag_root=" +if defined RUNFILES_MANIFEST_FILE if exist "%RUNFILES_MANIFEST_FILE%" ( + set "diag_manifest=%RUNFILES_MANIFEST_FILE%" + set "diag_reason=existing_environment_manifest" +) +if not defined diag_manifest for %%M in ("%~f0.runfiles_manifest" "%~dpn0.runfiles_manifest" "%~f0.runfiles\MANIFEST" "%~dpn0.runfiles\MANIFEST") do ( + if not defined diag_manifest if exist "%%~M" ( + set "diag_manifest=%%~M" + set "diag_reason=existing_adjacent_manifest" + ) +) +if not defined diag_manifest for %%R in ("%RUNFILES_DIR%" "%TEST_SRCDIR%") do ( + if not defined diag_manifest if not "%%~R"=="" if exist "%%~R\MANIFEST" ( + set "diag_manifest=%%~R\MANIFEST" + set "diag_reason=existing_directory_manifest" + ) +) +if not defined diag_manifest for %%R in ("%RUNFILES_DIR%" "%TEST_SRCDIR%" "%~f0.runfiles" "%~dpn0.runfiles") do ( + if not defined diag_root if not "%%~R"=="" if exist "%%~R\." ( + set "diag_root=%%~R" + set "diag_reason=existing_runfiles_directory" + ) +) +set "RUNFILES_MANIFEST_FILE=!diag_manifest!" +set "RUNFILES_DIR=!diag_root!" +set "TEST_SRCDIR=!diag_root!" +set "VOICE_DIAGNOSTIC_RUNFILES_REASON=!diag_reason!" +if not defined diag_manifest if not defined diag_root ( + >&2 echo voice diagnostic bootstrap: no existing manifest or runfiles directory + exit /b 1 +) + +call :resolve "python" "__PYTHON__" +if errorlevel 1 exit /b 1 +set "diag_python=!diag_result!" +call :resolve "driver" "__DRIVER__" +if errorlevel 1 exit /b 1 +set "diag_driver=!diag_result!" +call :resolve "config" "__CONFIG__" +if errorlevel 1 exit /b 1 +set "diag_config=!diag_result!" +"!diag_python!" "!diag_driver!" "!diag_config!" +exit /b !ERRORLEVEL! + +:resolve +set "diag_result=" +set "diag_role=%~1" +set "diag_key=%~2" +if defined diag_manifest ( + rem Read the file directly: nested type/findstr commands break on Windows CI. + for /f "usebackq tokens=1,*" %%A in ("!diag_manifest!") do ( + if "%%A"=="!diag_key!" ( + set "diag_result=%%B" + if not defined diag_result set "diag_result=%%A" + goto :resolved + ) + ) +) else ( + set "diag_native_key=!diag_key:/=\!" + set "diag_result=!diag_root!\!diag_native_key!" +) +:resolved +if not defined diag_result ( + >&2 echo voice diagnostic bootstrap: missing !diag_role! runfile !diag_key! + exit /b 1 +) +if not exist "!diag_result!" ( + >&2 echo voice diagnostic bootstrap: !diag_role! runfile does not exist: !diag_key! + exit /b 1 +) +exit /b 0 diff --git a/third_party/voice/windows_runtime_diagnostic.bzl b/third_party/voice/windows_runtime_diagnostic.bzl index a32688bfbe59..135db6de28c0 100644 --- a/third_party/voice/windows_runtime_diagnostic.bzl +++ b/third_party/voice/windows_runtime_diagnostic.bzl @@ -24,17 +24,16 @@ def _impl(ctx): })) launcher = ctx.actions.declare_file(ctx.label.name + ".bat") - # Windows manifest-only runfiles cannot be addressed by concatenating a root. - # Bootstrap just the declared Python/script/config entries via exact keys. - lines = ["@echo off", "setlocal"] - for name, file in [("PYTHON", tools.python.interpreter), ("DRIVER", ctx.file._driver), ("CONFIG", config)]: - key = _logical(ctx, file) - lines.extend([ - 'set "DIAG_%s=%%RUNFILES_DIR%%/%s"' % (name, key), - 'if defined RUNFILES_MANIFEST_FILE for /f "tokens=1,*" %%A in (\'type "%RUNFILES_MANIFEST_FILE%"\') do if "%%A"=="' + key + '" set "DIAG_' + name + '=%%B"', - ]) - lines.extend(['"%DIAG_PYTHON%" "%DIAG_DRIVER%" "%DIAG_CONFIG%"', "exit /b %ERRORLEVEL%"]) - ctx.actions.write(launcher, "\r\n".join(lines) + "\r\n", is_executable = True) + ctx.actions.expand_template( + template = ctx.file._launcher_template, + output = launcher, + substitutions = { + "__PYTHON__": _logical(ctx, tools.python.interpreter), + "__DRIVER__": _logical(ctx, ctx.file._driver), + "__CONFIG__": _logical(ctx, config), + }, + is_executable = True, + ) runfiles = ctx.runfiles(files = [config, ctx.file._driver, ctx.info_file, helper, unit], transitive_files = tools.files) for target in [ctx.attr.helper, ctx.attr.unit, ctx.attr._runtime]: runfiles = runfiles.merge(target[DefaultInfo].default_runfiles) @@ -48,6 +47,7 @@ windows_runtime_diagnostic_test = rule( "unit": attr.label(mandatory = True, executable = True, cfg = "target"), "_runtime": attr.label(default = "//third_party/voice:native_link_windows_x86_64"), "_tools": attr.label(default = "//third_party/voice:windows_tools_x86_64", cfg = "exec", providers = [WindowsBuildToolsInfo]), + "_launcher_template": attr.label(default = "//third_party/voice:windows_runtime_diagnostic.bat.tpl", allow_single_file = True), "_driver": attr.label(default = "//third_party/voice:windows_runtime_diagnostic.py", allow_single_file = True), }, ) diff --git a/third_party/voice/windows_runtime_diagnostic.py b/third_party/voice/windows_runtime_diagnostic.py index 3825e719f5ed..61105aa7c3fc 100644 --- a/third_party/voice/windows_runtime_diagnostic.py +++ b/third_party/voice/windows_runtime_diagnostic.py @@ -244,6 +244,7 @@ def main(): output = { "checkout_head": env.get("VOICE_DIAGNOSTIC_HEAD"), "checkout_tree": env.get("VOICE_DIAGNOSTIC_TREE"), + "runfiles_normalization_reason": env.get("VOICE_DIAGNOSTIC_RUNFILES_REASON"), "schema": 1, "github": { key: env.get(key) From a4a62b3b0cc6955980b7c55fdf4b4c8c22b6ace4 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 02:46:53 -0400 Subject: [PATCH 07/41] Refs #71: record actual Bazel paths and cache archive capacity --- .github/scripts/hosted_storage.py | 111 ++++++++++- .github/scripts/run-bazel-ci.sh | 1 + .github/scripts/test_hosted_storage.py | 263 ++++++++++++++++++++++++- .github/workflows/bazel.yml | 75 +++++++ 4 files changed, 436 insertions(+), 14 deletions(-) diff --git a/.github/scripts/hosted_storage.py b/.github/scripts/hosted_storage.py index bc7b3b4e7d35..b271e10c98aa 100644 --- a/.github/scripts/hosted_storage.py +++ b/.github/scripts/hosted_storage.py @@ -4,6 +4,8 @@ import json import os from pathlib import Path +import selectors +import signal import subprocess import sys import time @@ -12,6 +14,7 @@ from run_bazel_with_buildbuddy import ( REMOTE_EXECUTION_CONFIGS, bazel_args_without_remote_execution, + bazel_command, ) FORK = "CompleteDotTech/codex-subagents-multi-provider" @@ -37,6 +40,7 @@ def scoped(env): and env.get("RUNNER_ENVIRONMENT") == "github-hosted" and env.get("RUNNER_OS") in ("Linux", "macOS") and not env.get("BUILDBUDDY_API_KEY") + and env.get("HOSTED_STORAGE_REMOTE_CREDENTIALS") != "true" ) @@ -179,7 +183,85 @@ def measure(path): return result -def receipt(phase, args, env, status=None): +def bounded_query(command, env, timeout=15, limit=16384): + """Bound the query client, stdout and pipe lifetime on Linux and macOS.""" + child = subprocess.Popen( + command, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + data = bytearray() + deadline = time.monotonic() + timeout + finished = False + try: + with selectors.DefaultSelector() as selector: + selector.register(child.stdout, selectors.EVENT_READ) + while True: + remaining = deadline - time.monotonic() + if remaining <= 0 or not selector.select(remaining): + return "timeout", None + chunk = os.read(child.stdout.fileno(), 4096) + if not chunk: + break + if len(data) + len(chunk) > limit: + return "output-limit", None + data.extend(chunk) + code = child.wait(timeout=max(0.01, deadline - time.monotonic())) + finished = True + return ("ok", bytes(data)) if code == 0 else ("failed", None) + except subprocess.TimeoutExpired: + return "timeout", None + finally: + try: + if not finished: + # Bazelisk may spawn a client. The new process group belongs + # only to this query, not the existing build server/session. + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + if child.poll() is None: + child.wait(timeout=2) + finally: + child.stdout.close() + + +def actual_paths(args, env): + if not scoped(env): + return "out-of-scope", None + keys = ("output_base", "output_path", "execution_root") + # Same backend/startup normalization, cwd and effective local configuration + # as BUILD. Only safe info-compatible options, never arbitrary build flags. + command = bazel_command( + "--noexperimental_remote_repo_contents_cache", + "info", + *info_options(effective_arguments(args, env)), + *keys, + env=env, + ) + try: + state, raw = bounded_query(command, env) + if state != "ok": + return state, None + values = {} + for line in raw.decode("utf-8").splitlines(): + key, separator, value = line.partition(": ") + if ( + key not in keys + or not separator + or key in values + or not Path(value).is_absolute() + ): + return "invalid-output", None + values[key] = value + return ("ok", values) if set(values) == set(keys) else ("incomplete", None) + except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired): + return "unavailable", None + + +def receipt(phase, args, env, status=None, verified=None, query_state=None): output = { "schema": 1, "phase": phase, @@ -192,12 +274,20 @@ def receipt(phase, args, env, status=None): "run_id": env.get("GITHUB_RUN_ID"), "attempt": env.get("GITHUB_RUN_ATTEMPT"), "job": env.get("GITHUB_JOB"), - "paths_are": "declared; rc/config overrides unverified", + "paths_are": "actual bounded bazel info" + if verified + else "declared; rc/config overrides unverified", + "query_state": query_state, + "cache_save_requested": env.get("HOSTED_STORAGE_CACHE_SAVE") == "true" + if "HOSTED_STORAGE_CACHE_SAVE" in env + else None, "cpu_count": os.cpu_count(), "roots": {}, } seen = {} - for role, raw in roots(args, env).items(): + # A path-query receipt contains only the three verified roots; other phases + # retain the seven declared roles. Never sum overlapping directory totals. + for role, raw in (verified if verified else roots(args, env)).items(): if not raw: output["roots"][role] = {"disabled_or_unavailable": True} continue @@ -252,6 +342,13 @@ def main(): return if not scoped(env): return + if env.get("GITHUB_OUTPUT"): + try: + with open(env["GITHUB_OUTPUT"], "a", encoding="utf-8") as stream: + stream.write("enabled=true\n") + except OSError: + # An exhausted output filesystem must not prevent stdout metrics. + pass try: phase, status, *args = args if phase not in ( @@ -259,9 +356,15 @@ def main(): "after-build", "before-restore", "after-restore", + "before-cache-save", + "after-cache-save", + "after-build-paths", ): raise ValueError("unknown phase") - output = receipt(phase, args, env, None if status == "none" else int(status)) + state, verified = actual_paths(args, env) if mode == "resolve" else (None, None) + output = receipt( + phase, args, env, None if status == "none" else int(status), verified, state + ) text = json.dumps(output, sort_keys=True) if len(text.encode()) > 32768: raise ValueError("receipt bound exceeded") diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index e1c41fc863c4..a0c224c5b694 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -594,6 +594,7 @@ set -e # Capture filesystem state before any follow-up Bazel information request. if [[ $hosted_storage -eq 1 ]]; then python3 "$storage_script" observe after-build "$bazel_status" "${bazel_run_args[@]}" || echo "Hosted storage observation unavailable" >&2 + python3 "$storage_script" resolve after-build-paths "$bazel_status" "${bazel_run_args[@]}" || echo "Hosted Bazel paths unavailable" >&2 fi if [[ ${bazel_status:-0} -ne 0 ]]; then diff --git a/.github/scripts/test_hosted_storage.py b/.github/scripts/test_hosted_storage.py index 021c8ab6b2bf..72e5fceeea84 100644 --- a/.github/scripts/test_hosted_storage.py +++ b/.github/scripts/test_hosted_storage.py @@ -5,13 +5,14 @@ import json import os from pathlib import Path +import re import shutil import subprocess import sys import tempfile from types import SimpleNamespace import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch import hosted_storage as storage import run_bazel_with_buildbuddy as backend @@ -258,7 +259,246 @@ def test_info_projection_preserves_order_and_actual_configuration(self): ) +class ActualPathTests(unittest.TestCase): + def test_query_client_nonzero_exit_is_failure_without_output_payload(self): + child = Mock() + child.poll.return_value = 9 + child.wait.return_value = 9 + selector = Mock() + selector.select.return_value = [object()] + manager = Mock() + manager.__enter__ = Mock(return_value=selector) + manager.__exit__ = Mock(return_value=False) + with ( + patch.object(storage.subprocess, "Popen", return_value=child), + patch.object(storage.selectors, "DefaultSelector", return_value=manager), + patch.object(storage.os, "read", return_value=b""), + ): + self.assertEqual( + storage.bounded_query(["fake-bazel"], {}), ("failed", None) + ) + child.kill.assert_not_called() + child.stdout.close.assert_called_once_with() + + def test_actual_backend_query_uses_build_startup_and_effective_configuration(self): + with tempfile.TemporaryDirectory() as directory: + paths = { + key: str(Path(directory) / key) + for key in ("output_base", "output_path", "execution_root") + } + raw = "".join( + key + ": " + value + "\n" for key, value in paths.items() + ).encode() + env = dict(ENV, BAZEL_OUTPUT_USER_ROOT=directory) + with patch.object( + storage, "bounded_query", return_value=("ok", raw) + ) as query: + state, found = storage.actual_paths( + [ + "test", + "--config", + "ci-linux", + "-c", + "opt", + "--platforms", + "//:custom", + "--", + "--test_env=SECRET", + ], + env, + ) + self.assertEqual((state, found), ("ok", paths)) + command, actual_env = query.call_args.args + self.assertIs(actual_env, env) + self.assertIn("--output_user_root=" + directory, command) + self.assertIn("--noexperimental_remote_repo_contents_cache", command) + self.assertIn("--compilation_mode=opt", command) + self.assertIn("--platforms=//:custom", command) + self.assertIn("--disk_cache=", command) + self.assertNotIn("ci-linux", command) + self.assertFalse(any("SECRET" in arg for arg in command)) + + def test_query_failures_do_not_invent_actual_or_zero_metrics(self): + for failure in ("timeout", "failed", "output-limit"): + with patch.object(storage, "bounded_query", return_value=(failure, None)): + self.assertEqual(storage.actual_paths(["test"], ENV), (failure, None)) + for raw in ( + b"SECRET ERROR", + b"output_base: relative\n", + b"output_base: /a\noutput_base: /b\n", + ): + with patch.object(storage, "bounded_query", return_value=("ok", raw)): + state, found = storage.actual_paths(["test"], ENV) + self.assertNotEqual(state, "ok") + self.assertIsNone(found) + + def test_query_cannot_run_outside_hosted_keyless_scope(self): + for change in ( + {"RUNNER_OS": "Windows"}, + {"BUILDBUDDY_API_KEY": "synthetic"}, + {"GITHUB_REPOSITORY": "openai/codex"}, + {"RUNNER_ENVIRONMENT": "self-hosted"}, + {"HOSTED_STORAGE_REMOTE_CREDENTIALS": "true"}, + ): + with patch.object(storage, "bounded_query") as query: + self.assertEqual( + storage.actual_paths(["test"], dict(ENV, **change)), + ("out-of-scope", None), + ) + query.assert_not_called() + + def test_query_timeout_and_output_limit_clean_up_only_owned_client(self): + for ready, chunk, expected in ( + ([], b"", "timeout"), + ([object()], b"x" * 17, "output-limit"), + ): + child = Mock() + child.poll.return_value = None + selector = Mock() + selector.select.return_value = ready + manager = Mock() + manager.__enter__ = Mock(return_value=selector) + manager.__exit__ = Mock(return_value=False) + with ( + patch.object(storage.subprocess, "Popen", return_value=child) as spawn, + patch.object( + storage.selectors, "DefaultSelector", return_value=manager + ), + patch.object(storage.os, "read", return_value=chunk), + patch.object(storage.os, "killpg", create=True) as kill_group, + patch.object(storage.signal, "SIGKILL", 9, create=True), + ): + self.assertEqual( + storage.bounded_query(["fake-bazel"], {}, limit=16), + (expected, None), + ) + kill_group.assert_called_once_with(child.pid, 9) + child.kill.assert_not_called() + child.wait.assert_called_once_with(timeout=2) + child.stdout.close.assert_called_once_with() + self.assertEqual(spawn.call_args.kwargs["stderr"], subprocess.DEVNULL) + self.assertTrue(spawn.call_args.kwargs["start_new_session"]) + + def test_verified_receipt_has_only_three_actual_roots_with_overlap(self): + with tempfile.TemporaryDirectory() as directory: + verified = dict( + output_base=directory, + output_path=str(Path(directory) / "out"), + execution_root=str(Path(directory) / "exec"), + ) + with ( + patch.object(storage, "measure", return_value={"available_bytes": 42}), + patch.object(storage.subprocess, "run", side_effect=OSError), + ): + result = storage.receipt( + "after-build-paths", ["test"], ENV, 37, verified, "ok" + ) + self.assertEqual(set(result["roots"]), set(verified)) + self.assertEqual(result["paths_are"], "actual bounded bazel info") + self.assertIn("output_base", result["roots"]["output_path"]["overlaps"]) + + +class WorkflowTests(unittest.TestCase): + def test_cache_save_and_upload_are_scoped_without_changing_save_gate(self): + workflow = (Path(__file__).parents[1] / "workflows/bazel.yml").read_text() + jobs = dict( + re.findall( + r"^ ([\w-]+):\n(.*?)(?=^ [\w-]+:|\Z)", + workflow.split("jobs:\n", 1)[1], + re.M | re.S, + ) + ) + for name, block in jobs.items(): + if name not in ("test", "clippy", "verify-release-build"): + self.assertNotIn("hosted_storage", block) + continue + before = block.index("- name: Observe storage before cache archive") + save = block.index("- name: Save bazel repository cache") + after = block.index("- name: Observe storage after cache archive") + upload = block.index("- name: Upload hosted storage observations") + self.assertLess(before, save) + self.assertLess(save, after) + self.assertLess(after, upload) + self.assertIn( + "if: always() && !cancelled() && steps.prepare_bazel.outputs.repository-cache-hit != 'true'", + block[save:after], + ) + self.assertIn("continue-on-error: true", block[save:after]) + self.assertIn("runner.environment == 'github-hosted'", block[before:save]) + self.assertIn( + "runner.os == 'Linux' || runner.os == 'macOS'", block[before:save] + ) + self.assertIn( + "github.repository == '" + storage.FORK + "'", block[before:save] + ) + self.assertIn( + "HOSTED_STORAGE_REMOTE_CREDENTIALS: ${{ secrets.BUILDBUDDY_API_KEY != '' }}", + block[before:save], + ) + self.assertIn( + "steps.hosted_storage.outputs.enabled == 'true'", block[after:upload] + ) + self.assertIn( + "steps.prepare_bazel.outputs.repository-cache-hit != 'true'", + block[after:upload], + ) + self.assertIn( + "steps.hosted_storage.outputs.enabled == 'true'", block[upload:] + ) + self.assertIn( + "actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f", + block[upload:], + ) + self.assertIn( + "path: ${{ runner.temp }}/hosted-storage/*.json", block[upload:] + ) + + class ObservationTests(unittest.TestCase): + def test_remote_credential_boolean_blocks_workflow_outputs_and_metrics(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "step-output" + with ( + patch.dict( + os.environ, + dict( + ENV, + HOSTED_STORAGE_REMOTE_CREDENTIALS="true", + GITHUB_OUTPUT=str(output), + ), + clear=True, + ), + patch.object( + sys, "argv", ["script", "observe", "before-cache-save", "none"] + ), + patch.object(storage, "receipt") as collect, + ): + storage.main() + collect.assert_not_called() + self.assertFalse(output.exists()) + + def test_step_output_failure_does_not_hide_numeric_summary(self): + with tempfile.TemporaryDirectory() as directory: + result = {"exit_code": 37, "roots": {"output_base": {"available_bytes": 7}}} + out = io.StringIO() + with ( + patch.dict( + os.environ, + dict(ENV, RUNNER_TEMP=directory, GITHUB_OUTPUT=directory), + clear=True, + ), + patch.object( + sys, "argv", ["script", "observe", "after-build", "37", "test"] + ), + patch.object(storage, "receipt", return_value=result), + contextlib.redirect_stdout(out), + ): + storage.main() + self.assertIn('"output_base": 7', out.getvalue()) + self.assertEqual( + len(list((Path(directory) / "hosted-storage").glob("*.json"))), 1 + ) + def test_numeric_capacity_and_bounded_du(self): fs = SimpleNamespace( f_blocks=100, f_bfree=30, f_bavail=20, f_frsize=4096, f_favail=12 @@ -372,14 +612,15 @@ def invoke(self, flags=(), extra=()): # Mock just filesystem observations; use real defaults/projection. (root / "hosted_storage.py").write_text( "import json, os, sys\nimport policy\n" - "if sys.argv[1] == 'observe':\n" + "if sys.argv[1] in ('observe', 'resolve'):\n" " with open(os.environ['RECORD'], 'a') as f: f.write(json.dumps(sys.argv[2:4])+'\\n')\n" + " if sys.argv[1] == 'resolve': sys.exit(41)\n" "else: policy.main()\n", encoding="utf-8", ) (root / "run_bazel_with_buildbuddy.py").write_text( "#!/usr/bin/env python3\nimport json, os, sys\n" - "from backend import REMOTE_EXECUTION_CONFIGS, bazel_args_without_remote_execution\n" + "from backend import REMOTE_EXECUTION_CONFIGS, bazel_args_without_remote_execution, bazel_command\n" "if __name__ == '__main__':\n" " with open(os.environ['RECORD'], 'a') as f: f.write(json.dumps(sys.argv[1:])+'\\n')\n" " if 'info' in sys.argv: print('/nonexistent-testlogs'); sys.exit(0)\n" @@ -423,11 +664,13 @@ def test_failure_snapshot_precedes_info_and_original_exit_survives(self): self.assertEqual(result.returncode, 37, result.stderr) self.assertEqual(calls[0], ["before-build", "none"]) self.assertEqual(calls[2], ["after-build", "37"]) - self.assertIn("info", calls[3]) + self.assertEqual(calls[3], ["after-build-paths", "37"]) + self.assertIn("Hosted Bazel paths unavailable", result.stderr) + self.assertIn("info", calls[4]) self.assertIn("--disk_cache=", calls[1]) - self.assertIn("--disk_cache=", calls[3]) - self.assertIn("--compilation_mode=opt", calls[3]) - self.assertIn("--platforms=//:custom", calls[3]) + self.assertIn("--disk_cache=", calls[4]) + self.assertIn("--compilation_mode=opt", calls[4]) + self.assertIn("--platforms=//:custom", calls[4]) def test_explicit_split_cache_and_payload_are_not_changed(self): result, calls = self.invoke( @@ -439,15 +682,15 @@ def test_explicit_split_cache_and_payload_are_not_changed(self): self.assertEqual( build[build.index("--") + 1 :], ["//target", "--disk_cache=payload"] ) - self.assertIn("--disk_cache=cache with spaces", calls[3]) - self.assertNotIn("--disk_cache=payload", calls[3]) + self.assertIn("--disk_cache=cache with spaces", calls[4]) + self.assertNotIn("--disk_cache=payload", calls[4]) def test_both_remote_ci_spellings_are_removed_before_shell_dispatch(self): for name in ("ci-linux", "ci-macos", "ci-v8"): for flags in (["--config=" + name], ["--config", name]): result, calls = self.invoke(flags, ["--config=" + name]) self.assertEqual(result.returncode, 37, result.stderr) - build, info = calls[1], calls[3] + build, info = calls[1], calls[4] self.assertIn("--disk_cache=", build) self.assertIn("--disk_cache=", info) self.assertNotIn("--config=" + name, build[: build.index("--")]) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index c33dafe85734..4f56122c86c7 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -127,6 +127,16 @@ jobs: # Save the job-scoped Bazel repository cache after cache misses. Keep the # upload non-fatal so cache service issues never fail the job itself. + - name: Observe storage before cache archive + id: hosted_storage + if: always() && !cancelled() && runner.environment == 'github-hosted' && (runner.os == 'Linux' || runner.os == 'macOS') && github.repository == 'CompleteDotTech/codex-subagents-multi-provider' + continue-on-error: true + env: + HOSTED_STORAGE_REMOTE_CREDENTIALS: ${{ secrets.BUILDBUDDY_API_KEY != '' }} + HOSTED_STORAGE_CACHE_SAVE: ${{ steps.prepare_bazel.outputs.repository-cache-hit != 'true' }} + shell: bash + run: python3 .github/scripts/hosted_storage.py observe before-cache-save none + - name: Save bazel repository cache if: always() && !cancelled() && steps.prepare_bazel.outputs.repository-cache-hit != 'true' continue-on-error: true @@ -135,6 +145,21 @@ jobs: path: ${{ steps.prepare_bazel.outputs.repository-cache-path }} key: ${{ steps.prepare_bazel.outputs.repository-cache-key }} + - name: Observe storage after cache archive + if: always() && !cancelled() && steps.hosted_storage.outputs.enabled == 'true' && steps.prepare_bazel.outputs.repository-cache-hit != 'true' + continue-on-error: true + shell: bash + run: python3 .github/scripts/hosted_storage.py observe after-cache-save none + + - name: Upload hosted storage observations + if: always() && !cancelled() && steps.hosted_storage.outputs.enabled == 'true' + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: hosted-storage-${{ github.job }}-${{ matrix.target }} + path: ${{ runner.temp }}/hosted-storage/*.json + if-no-files-found: ignore + - name: Check for a clean worktree if: always() && !cancelled() uses: ./.github/actions/check-clean-worktree @@ -452,6 +477,16 @@ jobs: # Save the job-scoped Bazel repository cache after cache misses. Keep the # upload non-fatal so cache service issues never fail the job itself. + - name: Observe storage before cache archive + id: hosted_storage + if: always() && !cancelled() && runner.environment == 'github-hosted' && (runner.os == 'Linux' || runner.os == 'macOS') && github.repository == 'CompleteDotTech/codex-subagents-multi-provider' + continue-on-error: true + env: + HOSTED_STORAGE_REMOTE_CREDENTIALS: ${{ secrets.BUILDBUDDY_API_KEY != '' }} + HOSTED_STORAGE_CACHE_SAVE: ${{ steps.prepare_bazel.outputs.repository-cache-hit != 'true' }} + shell: bash + run: python3 .github/scripts/hosted_storage.py observe before-cache-save none + - name: Save bazel repository cache if: always() && !cancelled() && steps.prepare_bazel.outputs.repository-cache-hit != 'true' continue-on-error: true @@ -460,6 +495,21 @@ jobs: path: ${{ steps.prepare_bazel.outputs.repository-cache-path }} key: ${{ steps.prepare_bazel.outputs.repository-cache-key }} + - name: Observe storage after cache archive + if: always() && !cancelled() && steps.hosted_storage.outputs.enabled == 'true' && steps.prepare_bazel.outputs.repository-cache-hit != 'true' + continue-on-error: true + shell: bash + run: python3 .github/scripts/hosted_storage.py observe after-cache-save none + + - name: Upload hosted storage observations + if: always() && !cancelled() && steps.hosted_storage.outputs.enabled == 'true' + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: hosted-storage-${{ github.job }}-${{ matrix.target }} + path: ${{ runner.temp }}/hosted-storage/*.json + if-no-files-found: ignore + - name: Check for a clean worktree if: always() && !cancelled() uses: ./.github/actions/check-clean-worktree @@ -582,6 +632,16 @@ jobs: # Save the job-scoped Bazel repository cache after cache misses. Keep the # upload non-fatal so cache service issues never fail the job itself. + - name: Observe storage before cache archive + id: hosted_storage + if: always() && !cancelled() && runner.environment == 'github-hosted' && (runner.os == 'Linux' || runner.os == 'macOS') && github.repository == 'CompleteDotTech/codex-subagents-multi-provider' + continue-on-error: true + env: + HOSTED_STORAGE_REMOTE_CREDENTIALS: ${{ secrets.BUILDBUDDY_API_KEY != '' }} + HOSTED_STORAGE_CACHE_SAVE: ${{ steps.prepare_bazel.outputs.repository-cache-hit != 'true' }} + shell: bash + run: python3 .github/scripts/hosted_storage.py observe before-cache-save none + - name: Save bazel repository cache if: always() && !cancelled() && steps.prepare_bazel.outputs.repository-cache-hit != 'true' continue-on-error: true @@ -590,6 +650,21 @@ jobs: path: ${{ steps.prepare_bazel.outputs.repository-cache-path }} key: ${{ steps.prepare_bazel.outputs.repository-cache-key }} + - name: Observe storage after cache archive + if: always() && !cancelled() && steps.hosted_storage.outputs.enabled == 'true' && steps.prepare_bazel.outputs.repository-cache-hit != 'true' + continue-on-error: true + shell: bash + run: python3 .github/scripts/hosted_storage.py observe after-cache-save none + + - name: Upload hosted storage observations + if: always() && !cancelled() && steps.hosted_storage.outputs.enabled == 'true' + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: hosted-storage-${{ github.job }}-${{ matrix.target }} + path: ${{ runner.temp }}/hosted-storage/*.json + if-no-files-found: ignore + - name: Check for a clean worktree if: always() && !cancelled() uses: ./.github/actions/check-clean-worktree From a3b448f968bee7318767bb2bfa4a6a5b679f7ae9 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 03:01:38 -0400 Subject: [PATCH 08/41] Refs #71: propagate credential presence to restore observations --- .../build-bazel-package-binaries/action.yml | 5 + .../actions/build-codex-packages/action.yml | 5 + .github/actions/prepare-bazel-ci/action.yml | 8 ++ .../run-argument-comment-lint/action.yml | 1 + .github/scripts/test_hosted_storage.py | 114 ++++++++++++++++++ .github/workflows/bazel.yml | 5 + .github/workflows/macos-pty-probe.yml | 1 + .github/workflows/rust-release-windows.yml | 2 + .github/workflows/rust-release.yml | 2 + .github/workflows/rusty-v8-release.yml | 1 + .github/workflows/sdk.yml | 1 + .github/workflows/v8-canary.yml | 1 + 12 files changed, 146 insertions(+) diff --git a/.github/actions/build-bazel-package-binaries/action.yml b/.github/actions/build-bazel-package-binaries/action.yml index ae8660bf248c..2c058d87511b 100644 --- a/.github/actions/build-bazel-package-binaries/action.yml +++ b/.github/actions/build-bazel-package-binaries/action.yml @@ -1,6 +1,10 @@ name: Build Bazel package binaries description: Set up Bazel and stage unstripped runtime package binaries and symbols. inputs: + remote-credentials: + description: Boolean credential availability; callers must explicitly report whether their Bazel credential is configured. + required: false + default: "false" target: description: Rust release target triple. required: true @@ -14,6 +18,7 @@ runs: - name: Set up Bazel uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ inputs.remote-credentials }} target: ${{ inputs.target }} cache-scope: bazel-release diff --git a/.github/actions/build-codex-packages/action.yml b/.github/actions/build-codex-packages/action.yml index dea7cb623fef..e198adac6e8b 100644 --- a/.github/actions/build-codex-packages/action.yml +++ b/.github/actions/build-codex-packages/action.yml @@ -1,6 +1,10 @@ name: build-codex-packages description: Build and smoke-test unsigned Codex CLI and app-server release packages. inputs: + remote-credentials: + description: Boolean credential availability; callers must explicitly report whether their Bazel credential is configured. + required: false + default: "false" target: description: Rust release target triple to build and test. required: true @@ -77,6 +81,7 @@ runs: if: inputs.build-system == 'bazel' uses: ./.github/actions/build-bazel-package-binaries with: + remote-credentials: ${{ inputs.remote-credentials }} target: ${{ inputs.target }} build-mode: ${{ inputs.build-mode }} diff --git a/.github/actions/prepare-bazel-ci/action.yml b/.github/actions/prepare-bazel-ci/action.yml index 8892b6674e07..ee06f8764ce5 100644 --- a/.github/actions/prepare-bazel-ci/action.yml +++ b/.github/actions/prepare-bazel-ci/action.yml @@ -11,6 +11,10 @@ inputs: description: Provision pinned tools for the authorized native Windows fallback. required: false default: "false" + remote-credentials: + description: Boolean credential-availability receipt from the caller; external callers must report it explicitly rather than infer keylessness from the default. + required: false + default: "false" outputs: repository-cache-path: description: Filesystem path used for the Bazel repository cache. @@ -55,6 +59,8 @@ runs: - name: Observe storage before repository restore if: runner.environment == 'github-hosted' && (runner.os == 'Linux' || runner.os == 'macOS') && github.repository == 'CompleteDotTech/codex-subagents-multi-provider' continue-on-error: true + env: + HOSTED_STORAGE_REMOTE_CREDENTIALS: ${{ inputs.remote-credentials }} shell: bash run: python3 .github/scripts/hosted_storage.py observe before-restore none @@ -71,6 +77,8 @@ runs: - name: Observe storage after repository restore if: always() && !cancelled() && runner.environment == 'github-hosted' && (runner.os == 'Linux' || runner.os == 'macOS') && github.repository == 'CompleteDotTech/codex-subagents-multi-provider' continue-on-error: true + env: + HOSTED_STORAGE_REMOTE_CREDENTIALS: ${{ inputs.remote-credentials }} shell: bash run: python3 .github/scripts/hosted_storage.py observe after-restore none diff --git a/.github/actions/run-argument-comment-lint/action.yml b/.github/actions/run-argument-comment-lint/action.yml index 16cdd54eb47e..7211592b74a0 100644 --- a/.github/actions/run-argument-comment-lint/action.yml +++ b/.github/actions/run-argument-comment-lint/action.yml @@ -16,6 +16,7 @@ runs: - uses: ./.github/actions/prepare-bazel-ci with: target: ${{ inputs.target }} + remote-credentials: ${{ inputs.buildbuddy-api-key != '' }} cache-scope: bazel-argument-comment-lint - name: Install Linux sandbox build dependencies diff --git a/.github/scripts/test_hosted_storage.py b/.github/scripts/test_hosted_storage.py index 72e5fceeea84..46a9c699e031 100644 --- a/.github/scripts/test_hosted_storage.py +++ b/.github/scripts/test_hosted_storage.py @@ -454,6 +454,120 @@ def test_cache_save_and_upload_are_scoped_without_changing_save_gate(self): ) +class RestoreCredentialTests(unittest.TestCase): + def test_every_prepare_caller_explicitly_reports_credential_availability(self): + github = Path(__file__).parents[1] + found = [] + for directory in (github / "workflows", github / "actions"): + for path in sorted(directory.rglob("*.yml")): + content = path.read_text(encoding="utf-8") + calls = list( + re.finditer( + r"uses: \./\.github/actions/prepare-bazel-ci\n( +)with:\n((?:\1 [^\n]*\n)+)", + content, + ) + ) + self.assertEqual( + len(calls), + content.count("uses: ./.github/actions/prepare-bazel-ci"), + str(path), + ) + for call in calls: + params = call.group(2) + if directory.name == "workflows": + expected = "${{ secrets.BUILDBUDDY_API_KEY != '' }}" + elif path.parent.name == "run-argument-comment-lint": + expected = "${{ inputs.buildbuddy-api-key != '' }}" + else: + expected = "${{ inputs.remote-credentials }}" + self.assertIn("remote-credentials: " + expected, params, str(path)) + found.append(path) + self.assertTrue(found, "expected prepare-bazel-ci callers to audit") + package = (github / "actions/build-codex-packages/action.yml").read_text( + encoding="utf-8" + ) + self.assertRegex( + package, + r"uses: \./\.github/actions/build-bazel-package-binaries\n +with:\n +remote-credentials: \$\{\{ inputs.remote-credentials \}\}", + ) + + def test_composite_boolean_defaults_and_both_phase_env_connections(self): + actions = Path(__file__).parents[1] / "actions" + for name in ( + "prepare-bazel-ci", + "build-bazel-package-binaries", + "build-codex-packages", + ): + content = (actions / name / "action.yml").read_text(encoding="utf-8") + field = re.search( + r"^ remote-credentials:\n(.*?)(?=^ [\w-]+:|^outputs:|^runs:)", + content, + re.M | re.S, + ).group(1) + self.assertIn('default: "false"', field) + self.assertIn("required: false", field) + prepare = (actions / "prepare-bazel-ci/action.yml").read_text(encoding="utf-8") + steps = dict( + re.findall( + r"^ - name: ([^\n]+)\n(.*?)(?=^ - |\Z)", prepare, re.M | re.S + ) + ) + for when in ("before", "after"): + block = steps["Observe storage " + when + " repository restore"] + self.assertIn( + "HOSTED_STORAGE_REMOTE_CREDENTIALS: ${{ inputs.remote-credentials }}", + block, + ) + self.assertIn("observe " + when + "-restore none", block) + self.assertIn("continue-on-error: true", block) + self.assertNotIn("BUILDBUDDY_API_KEY:", block) + + def test_prepare_keyed_emits_nothing_and_keyless_retains_both_phases(self): + for available in ("true", "false"): + with ( + self.subTest(available=available), + tempfile.TemporaryDirectory() as directory, + ): + step_output = Path(directory) / "output" + env = dict( + ENV, + HOSTED_STORAGE_REMOTE_CREDENTIALS=available, + RUNNER_TEMP=directory, + GITHUB_OUTPUT=str(step_output), + ) + with ( + patch.dict(os.environ, env, clear=True), + patch.object( + storage, + "receipt", + return_value={"exit_code": None, "roots": {}}, + ) as collect, + contextlib.redirect_stdout(io.StringIO()), + ): + for phase in ("before-restore", "after-restore"): + with patch.object( + sys, "argv", ["script", "observe", phase, "none"] + ): + storage.main() + if available == "true": + collect.assert_not_called() + self.assertFalse(step_output.exists()) + self.assertFalse((Path(directory) / "hosted-storage").exists()) + else: + self.assertEqual( + [call.args[0] for call in collect.call_args_list], + ["before-restore", "after-restore"], + ) + self.assertEqual( + step_output.read_text().splitlines(), + ["enabled=true", "enabled=true"], + ) + self.assertEqual( + len(list((Path(directory) / "hosted-storage").glob("*.json"))), + 2, + ) + + class ObservationTests(unittest.TestCase): def test_remote_credential_boolean_blocks_workflow_outputs_and_metrics(self): with tempfile.TemporaryDirectory() as directory: diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 4f56122c86c7..ed397910a2cf 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -60,6 +60,7 @@ jobs: id: prepare_bazel uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: ${{ matrix.target }} cache-scope: bazel-${{ github.job }} @@ -207,6 +208,7 @@ jobs: id: prepare_bazel uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: x86_64-pc-windows-gnullvm native-windows-voice: ${{ github.repository == 'CompleteDotTech/codex-subagents-multi-provider' && secrets.BUILDBUDDY_API_KEY == '' }} # Reuse the former monolithic Windows test cache for restores. Do @@ -313,6 +315,7 @@ jobs: id: prepare_bazel uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: x86_64-pc-windows-msvc cache-scope: bazel-${{ github.job }} @@ -411,6 +414,7 @@ jobs: id: prepare_bazel uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: ${{ matrix.target }} cache-scope: bazel-${{ github.job }} native-windows-voice: ${{ github.repository == 'CompleteDotTech/codex-subagents-multi-provider' && matrix.target == 'x86_64-pc-windows-gnullvm' && secrets.BUILDBUDDY_API_KEY == '' }} @@ -549,6 +553,7 @@ jobs: id: prepare_bazel uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: ${{ matrix.target }} cache-scope: bazel-${{ github.job }} native-windows-voice: ${{ github.repository == 'CompleteDotTech/codex-subagents-multi-provider' && matrix.target == 'x86_64-pc-windows-gnullvm' && secrets.BUILDBUDDY_API_KEY == '' }} diff --git a/.github/workflows/macos-pty-probe.yml b/.github/workflows/macos-pty-probe.yml index cc20698aef29..fbcf6d0ee71a 100644 --- a/.github/workflows/macos-pty-probe.yml +++ b/.github/workflows/macos-pty-probe.yml @@ -41,6 +41,7 @@ jobs: id: prepare_bazel uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: ${{ steps.native.outputs.target }} cache-scope: macos-pty-probe diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index b91203f5d861..7fd8f95f74a6 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -108,6 +108,7 @@ jobs: if: ${{ matrix.build-system == 'bazel' }} uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: ${{ matrix.target }} cache-scope: bazel-release-${{ matrix.bundle }} @@ -212,6 +213,7 @@ jobs: - name: Prepare Bazel CI uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: ${{ matrix.target }} cache-scope: release-voice-windows - uses: ./.github/actions/setup-msvc-env diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index cf8127602b50..5534584104ce 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -272,6 +272,7 @@ jobs: if: matrix.build-system == 'bazel' || (matrix.bundle == 'primary' && contains(matrix.target, 'linux')) uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: ${{ matrix.target }} cache-scope: bazel-release-${{ matrix.bundle }} @@ -597,6 +598,7 @@ jobs: - name: Prepare Bazel CI uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: ${{ matrix.target }} cache-scope: bazel-release-primary - name: Build matching helper and native runtime diff --git a/.github/workflows/rusty-v8-release.yml b/.github/workflows/rusty-v8-release.yml index 3dcdbe09859d..b8109838cd8e 100644 --- a/.github/workflows/rusty-v8-release.yml +++ b/.github/workflows/rusty-v8-release.yml @@ -164,6 +164,7 @@ jobs: - name: Prepare Bazel CI uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: ${{ matrix.target }} cache-scope: bazel-v8 diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index 1155ecd6af68..5dc40b6c3d48 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -67,6 +67,7 @@ jobs: id: setup_bazel uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: x86_64-unknown-linux-gnu cache-scope: sdk diff --git a/.github/workflows/v8-canary.yml b/.github/workflows/v8-canary.yml index ae155133ed9c..f13b50658818 100644 --- a/.github/workflows/v8-canary.yml +++ b/.github/workflows/v8-canary.yml @@ -177,6 +177,7 @@ jobs: - name: Prepare Bazel CI uses: ./.github/actions/prepare-bazel-ci with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: ${{ matrix.target }} cache-scope: bazel-v8 From c1f5a9d47f4446bdb6c8f11a2bcd274249c6e785 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 03:28:52 -0400 Subject: [PATCH 09/41] Observe bounded hosted Windows voice diagnostic resources (refs #67) --- .github/actions/prepare-bazel-ci/action.yml | 5 + .github/actions/setup-bazel-ci/action.yml | 10 ++ .../scripts/observe_windows_ci_resources.ps1 | 82 ++++++++++ .../scripts/observe_windows_ci_resources.py | 154 ++++++++++++++++++ .../test_windows_ci_resource_observation.py | 132 +++++++++++++++ .github/workflows/bazel.yml | 14 +- 6 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/observe_windows_ci_resources.ps1 create mode 100644 .github/scripts/observe_windows_ci_resources.py create mode 100644 .github/scripts/test_windows_ci_resource_observation.py diff --git a/.github/actions/prepare-bazel-ci/action.yml b/.github/actions/prepare-bazel-ci/action.yml index 1a98e3bea483..032c3af736fc 100644 --- a/.github/actions/prepare-bazel-ci/action.yml +++ b/.github/actions/prepare-bazel-ci/action.yml @@ -11,6 +11,10 @@ inputs: description: Provision pinned tools for the authorized native Windows fallback. required: false default: "false" + windows-resource-observation: + description: Observe bounded hosted Windows diagnostic resources. + required: false + default: "false" outputs: repository-cache-path: description: Filesystem path used for the Bazel repository cache. @@ -36,6 +40,7 @@ runs: with: target: ${{ inputs.target }} native-windows-voice: ${{ inputs.native-windows-voice }} + windows-resource-observation: ${{ inputs.windows-resource-observation }} - name: Compute bazel repository cache key id: cache_bazel_repository_key diff --git a/.github/actions/setup-bazel-ci/action.yml b/.github/actions/setup-bazel-ci/action.yml index db6a09c86f61..77cec8c6a242 100644 --- a/.github/actions/setup-bazel-ci/action.yml +++ b/.github/actions/setup-bazel-ci/action.yml @@ -8,6 +8,10 @@ inputs: description: Provision pinned job-local tools for this fork's native Windows fallback. required: false default: "false" + windows-resource-observation: + description: Observe bounded hosted Windows diagnostic resources. + required: false + default: "false" outputs: repository-cache-path: description: Filesystem path used for the Bazel repository cache. @@ -19,6 +23,12 @@ runs: - id: setup_ci uses: ./.github/actions/setup-ci + - name: Observe resources after build path provisioning + if: inputs.windows-resource-observation == 'true' && runner.os == 'Windows' && runner.environment == 'github-hosted' && github.repository == 'CompleteDotTech/codex-subagents-multi-provider' + continue-on-error: true + shell: pwsh + run: python .github/scripts/observe_windows_ci_resources.py after-provision + - name: Set up Bazel uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # 0.19.0 # Without an explicit Bazelisk version, setup-bazel leaves PATH unchanged diff --git a/.github/scripts/observe_windows_ci_resources.ps1 b/.github/scripts/observe_windows_ci_resources.ps1 new file mode 100644 index 000000000000..dbf33066a27e --- /dev/null +++ b/.github/scripts/observe_windows_ci_resources.ps1 @@ -0,0 +1,82 @@ +# Numeric-only collector. Called by the bounded supervisor, never directly by CI. +param([int]$OwnedPid = 0, [double]$BuildStart = 0) +$ErrorActionPreference = 'Stop' +$result = [ordered]@{ unavailable = 0; truncated = $false; memory = $null; pagefiles = @(); configured_pagefiles = $null; volumes = @(); processes = @(); process_scope = 'live-owned-descendant-snapshot-only-detached-servers-unobserved'; vhd = $null } +function Number($Value, [long]$Scale = 1) { if ($null -eq $Value) { return $null }; return [long]$Value * $Scale } +function Query($Class, $Filter) { + $options = @{ ClassName = $Class; OperationTimeoutSec = 8; ErrorAction = 'Stop' } + if ($Filter) { $options.Filter = $Filter } + Get-CimInstance @options +} +try { + $m = Query Win32_PerfFormattedData_PerfOS_Memory $null + foreach ($field in @('AvailableBytes','CommittedBytes','CommitLimit','PercentCommittedBytesInUse')) { if ($null -eq $m.$field) { throw 'memory unavailable' } } + $result.memory = @{ available_bytes = [long]$m.AvailableBytes; committed_bytes = [long]$m.CommittedBytes; commit_limit_bytes = [long]$m.CommitLimit; percent_commit = [long]$m.PercentCommittedBytesInUse; commit_peak_bytes = $null } +} catch { $result.unavailable++ } +try { + $rows = @(Query Win32_PageFileUsage $null) + if ($rows.Count -gt 16) { $result.truncated = $true } + $result.pagefiles = @($rows | Select-Object -First 16 | ForEach-Object { @{ volume = [IO.Path]::GetPathRoot($_.Name); allocated_bytes = (Number $_.AllocatedBaseSize 1MB); current_bytes = (Number $_.CurrentUsage 1MB); peak_bytes = (Number $_.PeakUsage 1MB) } }) +} catch { $result.unavailable++ } +try { + $settings = @(Query Win32_PageFileSetting $null) + if ($settings.Count -gt 16) { $result.truncated = $true } + $result.configured_pagefiles = @($settings | Select-Object -First 16 | ForEach-Object { + @{ volume = [IO.Path]::GetPathRoot($_.Name); initial_bytes = $(if ($null -eq $_.InitialSize) { $null } else { [long]$_.InitialSize * 1MB }); maximum_bytes = $(if ($null -eq $_.MaximumSize) { $null } else { [long]$_.MaximumSize * 1MB }) } + }) +} catch { $result.unavailable++ } +$roots = [Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) +[void]$roots.Add('C:') +foreach ($name in @('CI_BUILD_ROOT','BAZEL_OUTPUT_BASE','BAZEL_OUTPUT_USER_ROOT','BAZEL_REPOSITORY_CACHE','BAZEL_REPO_CONTENTS_CACHE','CARGO_TARGET_DIR','RUNNER_TEMP','TEMP','TMP','VOICE_WINDOWS_BAZEL_REPOSITORY')) { + $path = [Environment]::GetEnvironmentVariable($name) + if ($path -and $path -match '^[A-Za-z]:[\\/]') { [void]$roots.Add([IO.Path]::GetPathRoot($path).TrimEnd('\')) } +} +if ($roots.Count -gt 16) { $result.truncated = $true } +foreach ($root in @($roots | Sort-Object | Select-Object -First 16)) { + try { + if ($root -notmatch '^[A-Za-z]:$') { throw 'unsupported volume' } + $d = Query Win32_LogicalDisk "DeviceID='$root'" + if ($null -eq $d -or $null -eq $d.FreeSpace -or $null -eq $d.Size) { throw 'unavailable volume' } + $result.volumes += @{ volume = $root; free_bytes = [long]$d.FreeSpace; total_bytes = [long]$d.Size } + } catch { $result.unavailable++; $result.volumes += @{ volume = $root; free_bytes = $null; total_bytes = $null } } +} +try { + if ($env:RUNNER_TEMP) { + $vhd = Join-Path $env:RUNNER_TEMP 'codex-dev-drive.vhdx' + if (Test-Path -LiteralPath $vhd -PathType Leaf) { + $result.vhd = @{ file_length_bytes = (Get-Item -LiteralPath $vhd).Length; logical_size_bytes = 68719476736; logical_size_source = 'setup-dev-drive.ps1'; backing_volume = [IO.Path]::GetPathRoot($vhd); allocated_physical_bytes = $null } + } + } +} catch { $result.unavailable++ } +# Query only descendants of the explicitly owned build, never all host processes. +if ($OwnedPid -gt 0) { + $queue = [Collections.Generic.Queue[int]]::new() + $seen = [Collections.Generic.HashSet[int]]::new() + try { + $root = Query Win32_Process "ProcessId=$OwnedPid" + $created = ([DateTimeOffset]$root.CreationDate).ToUnixTimeMilliseconds() / 1000.0 + if (-not $root -or [Math]::Abs($created - $BuildStart) -gt 2) { throw 'ownership unavailable' } + } catch { $result.unavailable++; $result | ConvertTo-Json -Depth 5 -Compress; exit 0 } + $creation = @{ $OwnedPid = $root.CreationDate } + $queue.Enqueue($OwnedPid) + while ($queue.Count -gt 0 -and $seen.Count -lt 128) { + $parent = $queue.Dequeue() + if (-not $seen.Add($parent)) { continue } + try { + $current = Query Win32_Process "ProcessId=$parent" + if (-not $current -or $current.CreationDate -ne $creation[$parent]) { $result.unavailable++; continue } + foreach ($p in @(Query Win32_Process "ParentProcessId=$parent")) { + if ($seen.Count + $queue.Count -ge 128) { $result.truncated = $true; break } + # A reused PID older than this build cannot be an owned child. + if (([DateTimeOffset]$p.CreationDate).ToUnixTimeMilliseconds() / 1000.0 -lt $created) { $result.unavailable++; continue } + $creation[[int]$p.ProcessId] = $p.CreationDate + $queue.Enqueue([int]$p.ProcessId) + if ($p.Name -match '^(rustc|link|cl|ninja|cmake|python|python3|bazel|bash)(\.exe)?$') { + $result.processes += @{ image = $p.Name.ToLowerInvariant(); working_bytes = (Number $p.WorkingSetSize); private_bytes = (Number $p.PrivatePageCount) } + } + } + } catch { $result.unavailable++ } + } + if ($queue.Count) { $result.truncated = $true } +} +$result | ConvertTo-Json -Depth 5 -Compress diff --git a/.github/scripts/observe_windows_ci_resources.py b/.github/scripts/observe_windows_ci_resources.py new file mode 100644 index 000000000000..d4c23eebb2a2 --- /dev/null +++ b/.github/scripts/observe_windows_ci_resources.py @@ -0,0 +1,154 @@ +"""Job-local, bounded observation; never changes resource budgets or host state.""" +import json +import os +from pathlib import Path +import platform +import subprocess +import sys +import time + +MAX_SAMPLE = 16 * 1024 +MAX_TOTAL = 2 * 1024 * 1024 +MAX_SAMPLES = 120 +INTERVAL = 60 + + +def authorized(env=None, system=None): + env = os.environ if env is None else env + return (env.get('GITHUB_REPOSITORY') == 'CompleteDotTech/codex-subagents-multi-provider' + and env.get('GITHUB_ACTIONS') == 'true' + and env.get('RUNNER_OS') == 'Windows' + and env.get('RUNNER_ENVIRONMENT') == 'github-hosted' + and (platform.system() if system is None else system) == 'Windows') + + +def identity(): + def git(ref): + try: + value = subprocess.check_output(['git', 'rev-parse', ref], timeout=2, + stderr=subprocess.DEVNULL, text=True).strip() + return value if len(value) == 40 and all(c in '0123456789abcdef' for c in value) else None + except (OSError, subprocess.SubprocessError): + return None + return dict(head=git('HEAD'), tree=git('HEAD^{tree}'), + run=os.environ.get('GITHUB_RUN_ID'), attempt=os.environ.get('GITHUB_RUN_ATTEMPT'), + job=os.environ.get('GITHUB_JOB'), architecture=platform.machine()) + + +def collect(owned_pid=0, start=0): + try: + completed = subprocess.run(['pwsh', '-NoProfile', '-NonInteractive', '-File', + str(Path(__file__).with_suffix('.ps1')), + '-OwnedPid', str(owned_pid), '-BuildStart', str(start)], timeout=10, + capture_output=True, text=True, check=True) + if len(completed.stdout.encode('utf-8')) > MAX_SAMPLE: + return dict(unavailable=1, truncated=True) + data = json.loads(completed.stdout) + if not isinstance(data, dict): + raise ValueError('invalid result') + return data + except (OSError, subprocess.SubprocessError, ValueError): + # Do not print exceptions: they may contain command or environment text. + return dict(unavailable=1, truncated=False) + + +class Observer: + def __init__(self): + self.provenance = identity() + self.path = Path(os.environ['RUNNER_TEMP']) / 'voice-runtime-diagnostic' / 'resources.jsonl' + self.count = 0 + self.size = 0 + self.exhausted = False + try: + self.size = self.path.stat().st_size + if self.size <= MAX_TOTAL: + with self.path.open('rb') as source: + self.count = sum(1 for _ in source) + else: + self.exhausted = True + except FileNotFoundError: + pass + except OSError: + self.exhausted = True + + def sample(self, phase, owned_pid=0, start=0): + if self.exhausted or self.count >= MAX_SAMPLES or self.size >= MAX_TOTAL: + print('windows-resources exhausted=1', flush=True) + return + data = collect(owned_pid, start) + record = dict(identity=self.provenance, phase=phase, unix_seconds=int(time.time()), **data) + line = json.dumps(record, separators=(',', ':')).encode('utf-8') + b'\n' + if len(line) > MAX_SAMPLE: + record = dict(identity=self.provenance, phase=phase, unix_seconds=int(time.time()), + unavailable=1, truncated=True) + line = json.dumps(record, separators=(',', ':')).encode('utf-8') + b'\n' + # Console precedes disk I/O, including when the runner disk is full. + memory = data.get('memory') or {} + numeric = {key: value for key, value in memory.items() + if value is None or type(value) in (int, float)} + print('windows-resources ' + json.dumps(dict(phase=phase, memory=numeric, + volumes=data.get('volumes', []), pagefiles=data.get('pagefiles', []), + unavailable=data.get('unavailable', 0), truncated=data.get('truncated', False)), + separators=(',', ':'))[:1900], flush=True) + self.count += 1 + if self.size + len(line) > MAX_TOTAL: + self.exhausted = True + return + try: + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open('ab') as target: + target.write(line) + target.flush() + self.size += len(line) + except OSError: + print('windows-resources artifact_unavailable=1', flush=True) + # Keep bounded console observations even if disk writes fail. + self.size += len(line) + + +def safe_sample(observer, phase, pid=0, start=0): + if observer is not None: + try: + observer.sample(phase, pid, start) + except Exception: + print('windows-resources unavailable=1', flush=True) + + +def supervise(command, observer): + safe_sample(observer, 'before-build') + # Argument vector, inherited console, no shell interpolation or output rewriting. + start = time.time() + child = subprocess.Popen(command) + deadline = time.monotonic() + INTERVAL + try: + while True: + try: + code = child.wait(timeout=max(0, deadline - time.monotonic())) + break + except subprocess.TimeoutExpired: + safe_sample(observer, 'build', child.pid, start) + deadline += INTERVAL + finally: + # No detached sampler exists. Collector subprocesses are synchronously + # reaped (and on timeout killed) by subprocess.run. Never kill host tools. + safe_sample(observer, 'build-exit') + return code + + +def main(args): + observer = None + if authorized(): + try: + observer = Observer() + except Exception: + print('windows-resources unavailable=1', flush=True) + if args[:2] == ['supervise', '--'] and len(args) > 2: + return supervise(args[2:], observer) + if len(args) == 1 and args[0] in ('before-prepare', 'after-provision', 'after-prepare', 'after-artifacts'): + safe_sample(observer, args[0]) + return 0 + raise SystemExit('expected phase or supervise -- command arguments') + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/.github/scripts/test_windows_ci_resource_observation.py b/.github/scripts/test_windows_ci_resource_observation.py new file mode 100644 index 000000000000..72217c42fcf2 --- /dev/null +++ b/.github/scripts/test_windows_ci_resource_observation.py @@ -0,0 +1,132 @@ +"""Fixture-only checks: no host CIM queries or actual builds.""" +import contextlib +import importlib.util +import io +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch, Mock + +SPEC = importlib.util.spec_from_file_location('observation', Path(__file__).with_name('observe_windows_ci_resources.py')) +m = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(m) + + +class ObservationTests(unittest.TestCase): + def test_scope_requires_every_gate_and_actual_windows(self): + env = dict(GITHUB_REPOSITORY='CompleteDotTech/codex-subagents-multi-provider', GITHUB_ACTIONS='true', RUNNER_OS='Windows', RUNNER_ENVIRONMENT='github-hosted') + self.assertTrue(m.authorized(env, 'Windows')) + for key in env: + altered = dict(env, **{key: 'other'}) + self.assertFalse(m.authorized(altered, 'Windows')) + self.assertFalse(m.authorized(env, 'Linux')) + + def test_timeout_and_errors_do_not_leak(self): + for error in (OSError('SECRET'), subprocess.TimeoutExpired('SECRET', 10), ValueError('SECRET')): + with patch.object(m.subprocess, 'run', side_effect=error): + result = m.collect(123, 1) + self.assertEqual(result['unavailable'], 1) + self.assertNotIn('SECRET', json.dumps(result)) + + def test_collection_deadline_and_no_shell(self): + with patch.object(m.subprocess, 'run', return_value=Mock(stdout='{}')) as run: + m.collect(123, 4) + self.assertEqual(run.call_args.kwargs['timeout'], 10) + self.assertNotIn('shell', run.call_args.kwargs) + self.assertEqual(run.call_args.args[0][-4:], ['-OwnedPid', '123', '-BuildStart', '4']) + + def test_build_exit_preserved_with_observer_failure(self): + observer = Mock() + observer.sample.side_effect = OSError('SECRET') + child = Mock() + child.wait.side_effect = [subprocess.TimeoutExpired('fixture', 60), 37] + child.pid = 123 + with patch.object(m.subprocess, 'Popen', return_value=child) as launch, contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(m.supervise(['bash', 'file with spaces', '--jobs=8'], observer), 37) + self.assertNotIn('SECRET', out.getvalue()) + launch.assert_called_once_with(['bash', 'file with spaces', '--jobs=8']) + child.kill.assert_not_called() + self.assertEqual(observer.sample.call_args_list[-1].args[0], 'build-exit') + + def test_start_failure_is_real_and_not_success(self): + with patch.object(m.subprocess, 'Popen', side_effect=FileNotFoundError('fixture')): + with self.assertRaises(FileNotFoundError): + m.supervise(['missing-build'], None) + + def test_oversized_collector_rejected(self): + with patch.object(m.subprocess, 'run', return_value=Mock(stdout='x' * (m.MAX_SAMPLE + 1))): + self.assertEqual(m.collect(), {'unavailable': 1, 'truncated': True}) + + def test_caps_and_stdout_precedes_write(self): + with tempfile.TemporaryDirectory() as temp, patch.dict(m.os.environ, {'RUNNER_TEMP': temp}), patch.object(m, 'identity', return_value={'head': 'fixture'}), patch.object(m, 'collect', return_value={'memory': {'available_bytes': 8}, 'unavailable': 0}): + observer = m.Observer() + with contextlib.redirect_stdout(io.StringIO()) as out: + observer.sample('before-prepare') + self.assertIn('available_bytes', out.getvalue()) + self.assertLess(observer.path.stat().st_size, m.MAX_SAMPLE) + observer.count = m.MAX_SAMPLES + with contextlib.redirect_stdout(io.StringIO()): + observer.sample('after-prepare') + self.assertEqual(len(observer.path.read_text().splitlines()), 1) + observer.count = 0 + observer.size = m.MAX_TOTAL + with contextlib.redirect_stdout(io.StringIO()): + observer.sample('after-prepare') + self.assertEqual(len(observer.path.read_text().splitlines()), 1) + + def test_workflow_only_diagnostic_job_uses_observer(self): + workflow = Path(__file__).parents[1] / 'workflows' / 'bazel.yml' + before, job = workflow.read_text().split(' voice-runtime-diagnostic:', 1) + self.assertNotIn('observe_windows_ci_resources', before) + self.assertIn('python .github/scripts/observe_windows_ci_resources.py supervise -- bash ./.github/scripts/run-bazel-ci.sh', job) + self.assertIn('if: github.event_name == \'workflow_dispatch\' && inputs.voice_runtime_diagnostic', job) + self.assertEqual(job.count('if: always() && !cancelled()'), 4) + self.assertNotIn('--jobs', job) + + def test_nested_observation_input_and_order(self): + actions = Path(__file__).parents[1] / 'actions' + prepare = (actions / 'prepare-bazel-ci' / 'action.yml').read_text() + setup = (actions / 'setup-bazel-ci' / 'action.yml').read_text() + for text in (prepare, setup): + self.assertIn('windows-resource-observation:', text) + self.assertIn('default: "false"', text.split('windows-resource-observation:', 1)[1]) + self.assertIn('windows-resource-observation: ${{ inputs.windows-resource-observation }}', prepare) + self.assertLess(setup.index('id: setup_ci'), setup.index('after-provision')) + self.assertLess(setup.index('after-provision'), setup.index('Provision pinned native Windows')) + workflow = (Path(__file__).parents[1] / 'workflows' / 'bazel.yml').read_text() + self.assertEqual(workflow.count('windows-resource-observation: true'), 1) + self.assertNotIn('windows-resource-observation', workflow.split(' voice-runtime-diagnostic:')[0]) + + def test_powershell_numeric_fixture_without_host_queries(self): + script = Path(__file__).with_name('observe_windows_ci_resources.ps1').resolve() + fixture = r''' +function global:Get-CimInstance { + param($ClassName, $Filter, $OperationTimeoutSec, $ErrorAction) + switch ($ClassName) { + 'Win32_PerfFormattedData_PerfOS_Memory' { [pscustomobject]@{ AvailableBytes=100; CommittedBytes=200; CommitLimit=400; PercentCommittedBytesInUse=50 } } + 'Win32_PageFileUsage' { 1..18 | ForEach-Object { [pscustomobject]@{ Name='C:\pagefile.sys'; AllocatedBaseSize=2; CurrentUsage=1; PeakUsage=2 } } } + 'Win32_LogicalDisk' { throw 'SECRET CIM failure' } + default { throw 'host query prohibited' } + } +} +$env:RUNNER_TEMP = '' +& $args[0] +''' + with tempfile.TemporaryDirectory() as temp: + runner = Path(temp) / 'fixture.ps1' + runner.write_text(fixture) + result = subprocess.run(['pwsh', '-NoProfile', '-File', str(runner), str(script)], capture_output=True, text=True, timeout=10, check=True) + data = json.loads(result.stdout) + self.assertEqual(data['memory']['committed_bytes'], 200) + self.assertEqual(data['pagefiles'][0]['allocated_bytes'], 2 * 1024 * 1024) + self.assertEqual(len(data['pagefiles']), 16) + self.assertTrue(data['truncated']) + self.assertIsNone(data['volumes'][0]['free_bytes']) + self.assertNotIn('SECRET', result.stdout) + self.assertLess(len(result.stdout.encode()), m.MAX_SAMPLE) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index b39691f99085..97629a7fa3b8 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -625,12 +625,20 @@ jobs: run: | "VOICE_DIAGNOSTIC_HEAD=$(git rev-parse HEAD)" >> $env:GITHUB_ENV "VOICE_DIAGNOSTIC_TREE=$(git rev-parse 'HEAD^{tree}')" >> $env:GITHUB_ENV + - name: Observe resources before preparation + shell: pwsh + run: python .github/scripts/observe_windows_ci_resources.py before-prepare - name: Prepare Bazel CI uses: ./.github/actions/prepare-bazel-ci with: target: x86_64-pc-windows-gnullvm native-windows-voice: true + windows-resource-observation: true cache-scope: bazel-test + - name: Observe resources after preparation and input installation + if: always() && !cancelled() + shell: pwsh + run: python .github/scripts/observe_windows_ci_resources.py after-prepare - name: Diagnose native MSVC voice runtime timeout-minutes: 45 shell: bash @@ -638,7 +646,7 @@ jobs: # its native MSVC route. This is not GNU cross-compile evidence. run: | unset BUILDBUDDY_API_KEY - ./.github/scripts/run-bazel-ci.sh \ + python .github/scripts/observe_windows_ci_resources.py supervise -- bash ./.github/scripts/run-bazel-ci.sh \ --print-failed-test-logs --print-failed-action-summary \ --windows-cross-compile -- test --nocache_test_results --test_output=all --zip_undeclared_test_outputs=false \ --test_env=VOICE_DIAGNOSTIC_HEAD --test_env=VOICE_DIAGNOSTIC_TREE \ @@ -654,6 +662,10 @@ jobs: for file in bazel-testlogs/codex-rs/voice-host/voice-host-windows-runtime-diagnostic-test/test.outputs/voice-runtime-diagnostic.json bazel-testlogs/codex-rs/voice-host/voice-host-windows-runtime-diagnostic-test/test.log; do if [[ -f "$file" ]]; then cp "$file" "${RUNNER_TEMP}/voice-runtime-diagnostic/"; fi done + - name: Observe resources after text artifact collection + if: always() && !cancelled() + shell: pwsh + run: python .github/scripts/observe_windows_ci_resources.py after-artifacts - name: Upload diagnostic text evidence if: always() && !cancelled() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 From 079d88273f158ea834fb43173c7030f112438729 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 03:36:05 -0400 Subject: [PATCH 10/41] fix(voice): package Windows helper and unit DLL siblings Keep raw Rust compilation and Clippy targets while exposing unchanged EXE bytes beside the full declared native DLL closure. Preserve startup diagnostics and verify both executable layouts and release staging paths. Refs #62. --- .github/workflows/rust-release-windows.yml | 2 +- codex-rs/voice-host/BUILD.bazel | 3 +- defs.bzl | 46 +++- third_party/voice/BUILD.bazel | 11 + .../voice/package_windows_executable.py | 82 +++++++ .../voice/test_package_windows_executable.py | 208 ++++++++++++++++++ .../voice/test_windows_runtime_diagnostic.py | 133 +++++++++-- third_party/voice/windows_executable.bzl | 66 ++++++ .../voice/windows_runtime_diagnostic.bzl | 13 +- .../voice/windows_runtime_diagnostic.py | 86 +++++++- 10 files changed, 621 insertions(+), 29 deletions(-) create mode 100644 third_party/voice/package_windows_executable.py create mode 100644 third_party/voice/test_package_windows_executable.py create mode 100644 third_party/voice/windows_executable.bzl diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index b91203f5d861..70c8f69a121a 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -296,7 +296,7 @@ jobs: output="voice-unsigned/${TARGET}" mkdir -p "$output" cp -R "$runtime" "$output/runtime" - cp bazel-bin/codex-rs/voice-host/codex-voice-host.exe "$output/codex-voice-host.exe" + cp bazel-bin/codex-rs/voice-host/codex-voice-host__windows-runtime/codex-voice-host.exe "$output/codex-voice-host.exe" chmod -R u+w "$output" tar -C "$output" -czf "voice-unsigned-${TARGET}.tar.gz" runtime codex-voice-host.exe - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 diff --git a/codex-rs/voice-host/BUILD.bazel b/codex-rs/voice-host/BUILD.bazel index 5ee6bd24d195..33fbe606bb13 100644 --- a/codex-rs/voice-host/BUILD.bazel +++ b/codex-rs/voice-host/BUILD.bazel @@ -4,6 +4,7 @@ load("//third_party/voice:windows_runtime_diagnostic.bzl", "windows_runtime_diag codex_rust_crate( name = "voice-host", binaries_with_build_commit = ["codex-voice-host"], + binary_windows_runtime = {"codex-voice-host": "//third_party/voice:native_link"}, crate_name = "codex_voice_host", crate_srcs = [], unit_test_args = ["--test-threads=1"], @@ -20,5 +21,5 @@ windows_runtime_diagnostic_test( "@platforms//cpu:x86_64", "@llvm//constraints/windows/abi:msvc", ], - unit = ":codex-voice-host-bin-unit-tests-bin", + unit = ":codex-voice-host-bin-unit-tests-bin__windows-runtime", ) diff --git a/defs.bzl b/defs.bzl index 164c772ae946..d2a51cad6efb 100644 --- a/defs.bzl +++ b/defs.bzl @@ -4,6 +4,7 @@ load("@rules_rust//cargo/private:cargo_build_script_wrapper.bzl", "cargo_build_s load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro", "rust_test") load("//bazel/rules/testing:foreign_platform_binary.bzl", "foreign_platform_binary") load("//bazel/rules/testing/wine:wine_runtime.bzl", "WINE_TEST_TARGET_COMPATIBLE_WITH", "wine_test_runtime") +load("//third_party/voice:windows_executable.bzl", "windows_voice_executable") # Reserve 8 MiB of stack on Windows and statically link the CRT on MSVC. # V8's x64 allocator shim defines malloc/free; importing them from dynamic UCRT @@ -209,6 +210,7 @@ def codex_rust_crate( lib_data_extra = [], rustc_flags_extra = [], binary_rustc_flags_extra = {}, + binary_windows_runtime = {}, binaries_with_build_commit = [], rustc_env = {}, rustc_env_files = [], @@ -255,6 +257,8 @@ def codex_rust_crate( lib_data_extra: Extra runtime data for the library target. binary_rustc_flags_extra: Mapping from binary names to extra rustc flags for those binary targets. + binary_windows_runtime: Opt-in binary name to declared voice runtime label, + packaged beside its EXE on Windows/MSVC only. Defaults to empty. binaries_with_build_commit: Binary names that embed STABLE_GIT_COMMIT from the generated build-commit environment file. Other workspace status is excluded from their compilation inputs. @@ -423,8 +427,11 @@ def codex_rust_crate( sanitized_binaries.append(binary) cargo_env_runfiles[":" + binary] = "CARGO_BIN_EXE_" + binary cargo_env["CARGO_BIN_EXE_" + binary] = "$(rlocationpath :%s)" % binary + runtime = binary_windows_runtime.get(binary) + raw_binary = binary + "__rust" if runtime else binary + binary_kwargs = {"binary_name": binary} if runtime else {} rust_binary( - name = binary, + name = raw_binary, crate_name = binary.replace("-", "_"), crate_root = main, deps = all_crate_deps() + maybe_deps + deps_extra, @@ -451,8 +458,24 @@ def codex_rust_crate( srcs = native.glob(["src/**/*.rs"]), stamp = 0, version = crate_version, - visibility = ["//visibility:public"], + visibility = ["//visibility:private"] if runtime else ["//visibility:public"], + **binary_kwargs ) + if runtime: + windows_voice_executable( + name = binary + "__windows-runtime", + binary = ":" + raw_binary, + runtime = runtime, + target_compatible_with = ["@platforms//os:windows", "@llvm//constraints/windows/abi:msvc"], + ) + native.alias( + name = binary, + actual = select({ + "//third_party/voice:windows_msvc": ":" + binary + "__windows-runtime", + "//conditions:default": ":" + raw_binary, + }), + visibility = ["//visibility:public"], + ) binary_unit_test_name = binary + "-bin-unit-tests" binary_unit_test_binary = binary_unit_test_name + "-bin" @@ -462,7 +485,7 @@ def codex_rust_crate( # sharding while Clippy can still discover the underlying test crate. rust_test( name = binary_unit_test_binary, - crate = ":" + binary, + crate = ":" + raw_binary, crate_features = crate_features, deps = all_crate_deps(normal_dev = True), rustc_flags = rustc_flags_extra + WINDOWS_RUSTC_LINK_FLAGS + [ @@ -475,6 +498,21 @@ def codex_rust_crate( version = crate_version, ) + exposed_unit = ":" + binary_unit_test_binary + if runtime: + windows_voice_executable( + name = binary_unit_test_binary + "__windows-runtime", + binary = exposed_unit, + runtime = runtime, + testonly = True, + tags = ["manual"], + target_compatible_with = ["@platforms//os:windows", "@llvm//constraints/windows/abi:msvc"], + ) + exposed_unit = select({ + "//third_party/voice:windows_msvc": exposed_unit + "__windows-runtime", + "//conditions:default": exposed_unit, + }) + binary_unit_test_kwargs = {} if unit_test_args: binary_unit_test_kwargs["args"] = unit_test_args @@ -487,7 +525,7 @@ def codex_rust_crate( workspace_root_test( name = binary_unit_test_name, env = test_env, - test_bin = ":" + binary_unit_test_binary, + test_bin = exposed_unit, workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", working_directory_marker = unit_test_working_directory_marker, target_compatible_with = binary_test_target_compatible_with, diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel index 69db38b19754..c6a8c5a651cd 100644 --- a/third_party/voice/BUILD.bazel +++ b/third_party/voice/BUILD.bazel @@ -8,6 +8,7 @@ load(":windows_native.bzl", "windows_build_tools", "windows_native_prefix") # All native build consumers must receive the same pinned source manifest. exports_files([ + "package_windows_executable.py", "opus-toolchain.cmake", "windows_runtime_diagnostic.bat.tpl", "windows_runtime_diagnostic.py", @@ -433,3 +434,13 @@ label_flag( ) for cpu in ("x86_64", "aarch64") ] + +# The public helper alias selects packaged executables only for this ABI. +config_setting( + name = "windows_msvc", + constraint_values = [ + "@platforms//os:windows", + "@llvm//constraints/windows/abi:msvc", + ], + visibility = ["//visibility:public"], +) diff --git a/third_party/voice/package_windows_executable.py b/third_party/voice/package_windows_executable.py new file mode 100644 index 000000000000..dc4c8da744a9 --- /dev/null +++ b/third_party/voice/package_windows_executable.py @@ -0,0 +1,82 @@ +"""Copy a declared Windows executable/DLL closure without modifying any bytes.""" + +import hashlib +import json +from pathlib import Path +import shutil +import sys + + +def digest(path): + value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def _basename(name): + if ( + not name + or any(character in name for character in '<>:"/\\|?*\0') + or name.rstrip(" .") != name + or name.split(".")[0].casefold() in {"con", "prn", "aux", "nul"} + ): + raise ValueError("invalid package basename") + return name.casefold() + + +def package(config): + root = Path(config["directory"]) + if root.is_symlink(): + raise ValueError("package directory cannot be a symlink") + existing = ( + {path.name.casefold() for path in root.iterdir()} if root.exists() else set() + ) + receipt = Path(config["receipt"]) + pairs = [config["executable"], *config["dlls"]] + if not config["dlls"]: + raise ValueError("declared DLL closure must not be empty") + names = set() + validated = [] + for index, pair in enumerate(pairs): + source, output = Path(pair["source"]), Path(pair["output"]) + key = _basename(output.name) + if key in names: + raise ValueError("case-insensitive package basename collision") + names.add(key) + suffix = ".exe" if index == 0 else ".dll" + if source.name != output.name or output.suffix.casefold() != suffix: + raise ValueError("package must preserve executable and DLL basenames") + if ".." in output.parts or output.parent.resolve() != root.resolve(): + raise ValueError("package output escapes its declared directory") + if source.resolve() == output.resolve(): + raise ValueError("package source and destination must differ") + if key in existing or output.exists() or output.is_symlink(): + raise ValueError("package output already exists") + if not source.is_file(): + raise ValueError("declared package input is missing") + validated.append((source, output, digest(source))) + if ".." in receipt.parts or receipt.parent.resolve() != root.resolve(): + raise ValueError("package receipt escapes its declared directory") + if _basename(receipt.name) in names: + raise ValueError("package receipt collides with a payload") + if receipt.name.casefold() in existing or receipt.exists() or receipt.is_symlink(): + raise ValueError("package receipt already exists") + root.mkdir(parents=True, exist_ok=True) + records = [] + for source, output, expected in validated: + with source.open("rb") as reader, output.open("xb") as writer: + shutil.copyfileobj(reader, writer, 1024 * 1024) + if digest(output) != expected: + raise ValueError("package byte checksum mismatch") + records.append( + {"name": output.name, "source_sha256": expected, "output_sha256": expected} + ) + with receipt.open("x", encoding="utf-8") as stream: + json.dump({"executable": records[0], "dlls": records[1:]}, stream, indent=2) + stream.write("\n") + + +if __name__ == "__main__": + package(json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))) diff --git a/third_party/voice/test_package_windows_executable.py b/third_party/voice/test_package_windows_executable.py new file mode 100644 index 000000000000..f644363873c1 --- /dev/null +++ b/third_party/voice/test_package_windows_executable.py @@ -0,0 +1,208 @@ +"""Declared payload copying, collision protection, and voice consumer contracts.""" + +import copy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import package_windows_executable as packaging + +DLL_NAMES = """gio-2.0-0.dll glib-2.0-0.dll gmodule-2.0-0.dll gobject-2.0-0.dll + gstapp-1.0-0.dll gstapp.dll gstaudio-1.0-0.dll gstaudioconvert.dll + gstaudioresample.dll gstbase-1.0-0.dll gstcoreelements.dll gstnet-1.0-0.dll + gstopus.dll gstpbutils-1.0-0.dll gstreamer-1.0-0.dll gstrtp-1.0-0.dll + gstrtp.dll gstrtpmanager.dll gsttag-1.0-0.dll gstvideo-1.0-0.dll intl-8.dll + libffi-8.dll opus.dll pcre2-8.dll z.dll""".split() + + +class PackageTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix="voice package ") + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.inputs = self.root / "declared inputs" + self.inputs.mkdir() + self.executable = self.inputs / "codex-voice-host.exe" + self.executable.write_bytes(b"unchanged executable\x00\xff") + self.dlls = [] + for name in DLL_NAMES: + path = self.inputs / name + path.write_bytes((name + "\x00payload").encode()) + self.dlls.append(path) + self.config = self.config_for("helper") + + def config_for(self, name, executable=None): + executable = executable or self.executable + directory = self.root / name + return { + "directory": str(directory), + "receipt": str(directory / "packaging.json"), + "executable": { + "source": str(executable), + "output": str(directory / executable.name), + }, + "dlls": [ + {"source": str(path), "output": str(directory / path.name)} + for path in self.dlls + ], + } + + def assert_no_payload_written(self, config): + directory = Path(config["directory"]) + self.assertFalse(any(directory.glob("*.exe"))) + self.assertFalse(any(directory.glob("*.dll"))) + + def test_two_unique_packages_keep_all_25_dlls_and_identical_bytes(self): + unit = self.inputs / "codex-voice-host-bin-unit-tests-bin.exe" + unit.write_bytes(b"unchanged unit executable") + for name, executable in [ + ("helper package", self.executable), + ("unit package", unit), + ]: + before = {path: path.read_bytes() for path in [executable, *self.dlls]} + config = self.config_for(name, executable) + packaging.package(config) + directory = Path(config["directory"]) + self.assertEqual(len(config["dlls"]), 25) + self.assertEqual( + {path.name for path in directory.iterdir()}, + {executable.name, "packaging.json", *DLL_NAMES}, + ) + receipt = json.loads( + (directory / "packaging.json").read_text(encoding="utf-8") + ) + self.assertEqual(len(receipt["dlls"]), 25) + for source, value in before.items(): + self.assertEqual(source.read_bytes(), value) + self.assertEqual((directory / source.name).read_bytes(), value) + self.assertEqual( + receipt["executable"]["source_sha256"], + packaging.digest(executable), + ) + self.assertEqual( + receipt["executable"]["source_sha256"], + receipt["executable"]["output_sha256"], + ) + self.assertTrue( + all( + row["source_sha256"] == row["output_sha256"] + for row in receipt["dlls"] + ) + ) + + def test_case_insensitive_collision_rejected_before_writes_even_same_bytes(self): + other = self.inputs / "GLIB-2.0-0.DLL" + other.write_bytes(self.dlls[1].read_bytes()) + self.config["dlls"].append( + { + "source": str(other), + "output": str(Path(self.config["directory"]) / other.name), + } + ) + with self.assertRaisesRegex(ValueError, "collision"): + packaging.package(self.config) + self.assert_no_payload_written(self.config) + + def test_duplicate_exact_input_mapping_is_not_silently_overwritten(self): + self.config["dlls"].append(dict(self.config["dlls"][0])) + with self.assertRaisesRegex(ValueError, "collision"): + packaging.package(self.config) + self.assert_no_payload_written(self.config) + + def test_preexisting_exe_dll_or_receipt_never_overwritten(self): + for target in [self.executable.name, DLL_NAMES[-1], "packaging.json"]: + with self.subTest(target=target): + config = self.config_for("preexisting-" + target) + directory = Path(config["directory"]) + directory.mkdir() + existing = directory / target + existing.write_bytes(b"preserve me") + with self.assertRaisesRegex(ValueError, "already exists"): + packaging.package(config) + self.assertEqual(list(directory.iterdir()), [existing]) + self.assertEqual(existing.read_bytes(), b"preserve me") + + def test_preexisting_case_alias_is_not_overwritten_on_case_sensitive_exec_hosts( + self, + ): + directory = Path(self.config["directory"]) + directory.mkdir() + existing = directory / DLL_NAMES[-1].upper() + existing.write_bytes(b"preserve case alias") + with self.assertRaisesRegex(ValueError, "already exists"): + packaging.package(self.config) + self.assertEqual(list(directory.iterdir()), [existing]) + self.assertEqual(existing.read_bytes(), b"preserve case alias") + + def test_missing_late_input_fails_before_first_output(self): + self.dlls[-1].unlink() + with self.assertRaisesRegex(ValueError, "input is missing"): + packaging.package(self.config) + self.assert_no_payload_written(self.config) + + def test_no_dlls_or_escaped_output_or_source_destination_alias(self): + empty = copy.deepcopy(self.config) + empty["dlls"] = [] + with self.assertRaisesRegex(ValueError, "must not be empty"): + packaging.package(empty) + escaped = copy.deepcopy(self.config) + escaped["dlls"][-1]["output"] = str(self.root / "outside" / DLL_NAMES[-1]) + with self.assertRaisesRegex(ValueError, "escapes"): + packaging.package(escaped) + alias = copy.deepcopy(self.config) + alias["directory"] = str(self.inputs) + alias["executable"]["output"] = str(self.executable) + with self.assertRaisesRegex(ValueError, "must differ"): + packaging.package(alias) + self.assert_no_payload_written(self.config) + + def test_changed_copy_bytes_fail_instead_of_publishing_success_receipt(self): + def corrupt(_source, output, _length): + output.write(b"changed bytes") + + with patch.object(packaging.shutil, "copyfileobj", side_effect=corrupt): + with self.assertRaisesRegex(ValueError, "checksum mismatch"): + packaging.package(self.config) + self.assertFalse(Path(self.config["receipt"]).exists()) + + +class ConsumerContractTests(unittest.TestCase): + def test_raw_compilation_and_packaged_execution_routes_remain_distinct(self): + root = Path(__file__).resolve().parents[2] + source = (root / "defs.bzl").read_text(encoding="utf-8") + self.assertIn("binary_windows_runtime = {}", source) + self.assertIn( + 'binary_kwargs = {"binary_name": binary} if runtime else {}', source + ) + self.assertIn('crate = ":" + raw_binary', source) + self.assertIn('"//conditions:default": ":" + raw_binary', source) + self.assertIn('"//conditions:default": exposed_unit', source) + self.assertIn("test_bin = exposed_unit", source) + self.assertIn( + 'cargo_env["CARGO_BIN_EXE_" + binary] = "$(rlocationpath :%s)" % binary', + source, + ) + voice = (root / "codex-rs/voice-host/BUILD.bazel").read_text(encoding="utf-8") + self.assertIn( + 'unit = ":codex-voice-host-bin-unit-tests-bin__windows-runtime"', voice + ) + windows = (root / ".github/workflows/rust-release-windows.yml").read_text( + encoding="utf-8" + ) + self.assertIn( + "cp bazel-bin/codex-rs/voice-host/codex-voice-host__windows-runtime/codex-voice-host.exe", + windows, + ) + macos = (root / ".github/workflows/rust-release.yml").read_text( + encoding="utf-8" + ) + self.assertIn( + 'cp bazel-bin/codex-rs/voice-host/codex-voice-host "$output/codex-voice-host"', + macos, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/third_party/voice/test_windows_runtime_diagnostic.py b/third_party/voice/test_windows_runtime_diagnostic.py index 4a59f886c813..905d8865283a 100644 --- a/third_party/voice/test_windows_runtime_diagnostic.py +++ b/third_party/voice/test_windows_runtime_diagnostic.py @@ -5,9 +5,58 @@ from unittest.mock import patch import windows_runtime_diagnostic as diagnostic +import package_windows_executable as packaging class DiagnosticTests(unittest.TestCase): + def packaged_fixture(self, root): + files = {} + runtime = root / "native" / "bin" + runtime.mkdir(parents=True) + dll = runtime / "voice.dll" + dll.write_bytes(b"declared DLL") + files["runtime-dll"] = dll + packages = {} + for name in ["helper", "unit"]: + source = root / (name + "-raw") / (name + ".exe") + source.parent.mkdir() + source.write_bytes(b"unchanged PE") + directory = root / (name + "-package") + packaging.package( + { + "directory": str(directory), + "executable": { + "source": str(source), + "output": str(directory / source.name), + }, + "dlls": [{"source": str(dll), "output": str(directory / dll.name)}], + "receipt": str(directory / "packaging.json"), + } + ) + files[name + ".exe"] = directory / source.name + files[name + "-raw"] = source + files[name + "-receipt"] = directory / "packaging.json" + files[name + "-dll"] = directory / dll.name + packages[name] = { + "raw": name + "-raw", + "receipt": name + "-receipt", + "dlls": [name + "-dll"], + } + for name in ["dumpbin.exe", "status"]: + files[name] = root / name + files[name].write_bytes(b"PE") + files["status"].write_text("STABLE_GIT_COMMIT " + "a" * 40 + "\n") + config = dict( + helper="helper.exe", + unit="unit.exe", + dumpbin="dumpbin.exe", + status="status", + runtime=["runtime-dll"], + packages=packages, + tool_environment={"TOOL_ONLY": "yes"}, + ) + return files, config + def test_exe_dll_and_deferred_imports(self): text = """File Type: EXECUTABLE IMAGE 8664 machine (x64) @@ -112,20 +161,7 @@ def test_timeout_and_bounded_output(self): def test_diagnose_preserves_source_and_copy_environment(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) - files = {} - for name in ["helper.exe", "unit.exe", "dumpbin.exe", "status"]: - path = root / name - path.write_bytes(b"PE") - files[name] = path - files["status"].write_text("STABLE_GIT_COMMIT " + "a" * 40 + "\n") - config = dict( - helper="helper.exe", - unit="unit.exe", - dumpbin="dumpbin.exe", - status="status", - runtime=[], - tool_environment={"TOOL_ONLY": "yes"}, - ) + files, config = self.packaged_fixture(root) env = {"TEST_TMPDIR": str(root)} calls = [] @@ -151,6 +187,75 @@ def runner(command, environment, cwd, *args): self.assertNotIn("TOOL_ONLY", calls[0][1]) self.assertIn("TOOL_ONLY", calls[-1][1]) self.assertEqual(output["helper_sha256"], output["installed_sha256"]) + self.assertTrue( + all(value["valid"] for value in output["packaging"].values()) + ) + self.assertEqual(output["packaging"]["helper"]["declared_dll_count"], 1) + + def test_package_check_rejects_missing_modified_dll_and_executable_bytes(self): + for change in ["missing", "DLL bytes", "EXE bytes", "receipt"]: + with ( + self.subTest(change=change), + tempfile.TemporaryDirectory() as directory, + ): + files, config = self.packaged_fixture(Path(directory)) + if change == "missing": + files["helper-dll"].unlink() + elif change == "DLL bytes": + files["helper-dll"].write_bytes(b"changed") + elif change == "EXE bytes": + files["helper.exe"].write_bytes(b"changed") + else: + files["helper-receipt"].write_text("{}") + with patch.object( + diagnostic, "resolve", side_effect=lambda key, _: files[key] + ): + with self.assertRaises((ValueError, KeyError)): + diagnostic.package_adjacency( + files["helper.exe"], + config["packages"]["helper"], + {"voice.dll": files["runtime-dll"]}, + {}, + ) + + def test_failed_package_check_retains_all_original_startup_and_pe_metrics(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + files, config = self.packaged_fixture(root) + files["helper-dll"].write_bytes(b"changed package DLL") + + def runner(command, environment, cwd, *args): + if "--build-commit" in command: + installed = "installed-fixture" in command[0] + return { + "code": 0 if installed else 0xC0000135, + "stdout": "a" * 40 if installed else "", + "stderr": "", + } + if "--list" in command: + return {"code": 0xC0000135, "stdout": "", "stderr": ""} + return { + "code": 0, + "stdout": "8664 machine (x64)\nFile Type: EXECUTABLE IMAGE", + "stderr": "", + } + + with ( + patch.object( + diagnostic, "resolve", side_effect=lambda key, _: files[key] + ), + patch.object(diagnostic, "run", side_effect=runner), + ): + output = {} + self.assertFalse( + diagnostic.diagnose(config, {"TEST_TMPDIR": str(root)}, output) + ) + self.assertEqual( + [row["code"] for row in output["probes"]], [0xC0000135, 0, 0xC0000135] + ) + self.assertEqual(len(output["pe"]), 3) + self.assertEqual(output["helper_sha256"], output["installed_sha256"]) + self.assertFalse(output["packaging"]["helper"]["valid"]) if __name__ == "__main__": diff --git a/third_party/voice/windows_executable.bzl b/third_party/voice/windows_executable.bzl new file mode 100644 index 000000000000..93147895de2c --- /dev/null +++ b/third_party/voice/windows_executable.bzl @@ -0,0 +1,66 @@ +"""Declared executable-adjacent voice DLLs for Windows/MSVC only.""" + +load("@rules_python//python:py_runtime_info.bzl", "PyRuntimeInfo") + +VoiceExecutablePackageInfo = provider(fields = ["raw_executable", "executable", "dlls", "receipt"]) + +def _basename_key(file): + return file.basename.lower() + +def _impl(ctx): + raw = ctx.executable.binary + runtime = ctx.attr.runtime[DefaultInfo] + dlls = sorted([file for file in runtime.default_runfiles.files.to_list() if file.basename.lower().endswith(".dll")], key = _basename_key) + if not dlls: + fail("Windows voice executable requires its declared DLL closure") + names = {raw.basename.lower(): raw} + for dll in dlls: + key = dll.basename.lower() + if key in names: + fail("Case-insensitive voice package basename collision: " + dll.basename) + names[key] = dll + executable = ctx.actions.declare_file(ctx.label.name + "/" + raw.basename) + copied = [ctx.actions.declare_file(ctx.label.name + "/" + dll.basename) for dll in dlls] + receipt = ctx.actions.declare_file(ctx.label.name + "/packaging.json") + config = ctx.actions.declare_file(ctx.label.name + ".copy.json") + ctx.actions.write(config, json.encode({ + "directory": executable.dirname, + "executable": {"source": raw.path, "output": executable.path}, + "dlls": [{"source": source.path, "output": output.path} for source, output in zip(dlls, copied)], + "receipt": receipt.path, + })) + python = ctx.attr._python[PyRuntimeInfo] + ctx.actions.run( + executable = python.interpreter, + arguments = [ctx.file._copy.path, config.path], + inputs = depset([raw, config, ctx.file._copy, python.interpreter] + dlls, transitive = [python.files]), + outputs = [executable, receipt] + copied, + mnemonic = "VoiceExecutablePackage", + ) + own = ctx.runfiles(files = [raw, executable, receipt] + copied) + binary = ctx.attr.binary[DefaultInfo] + providers = [ + DefaultInfo( + executable = executable, + files = depset([executable]), + default_runfiles = own.merge(binary.default_runfiles).merge(runtime.default_runfiles), + data_runfiles = own.merge(binary.data_runfiles).merge(runtime.data_runfiles), + ), + VoiceExecutablePackageInfo(raw_executable = raw, executable = executable, dlls = copied, receipt = receipt), + ] + if OutputGroupInfo in ctx.attr.binary: + providers.append(ctx.attr.binary[OutputGroupInfo]) + if RunEnvironmentInfo in ctx.attr.binary: + providers.append(ctx.attr.binary[RunEnvironmentInfo]) + return providers + +windows_voice_executable = rule( + implementation = _impl, + executable = True, + attrs = { + "binary": attr.label(mandatory = True, executable = True, cfg = "target"), + "runtime": attr.label(mandatory = True), + "_copy": attr.label(default = "//third_party/voice:package_windows_executable.py", allow_single_file = True), + "_python": attr.label(default = "@python_3_12//:py3_runtime", cfg = "exec"), + }, +) diff --git a/third_party/voice/windows_runtime_diagnostic.bzl b/third_party/voice/windows_runtime_diagnostic.bzl index 135db6de28c0..5fd88b889029 100644 --- a/third_party/voice/windows_runtime_diagnostic.bzl +++ b/third_party/voice/windows_runtime_diagnostic.bzl @@ -1,5 +1,6 @@ """Manual, observational Windows loader diagnostic; never packages the binaries.""" +load("//third_party/voice:windows_executable.bzl", "VoiceExecutablePackageInfo") load("//third_party/voice:windows_native.bzl", "WindowsBuildToolsInfo") def _logical(ctx, file): @@ -17,6 +18,14 @@ def _impl(ctx): ctx.actions.write(config, json.encode({ "helper": _logical(ctx, helper), "unit": _logical(ctx, unit), + "packages": { + name: { + "raw": _logical(ctx, target[VoiceExecutablePackageInfo].raw_executable), + "receipt": _logical(ctx, target[VoiceExecutablePackageInfo].receipt), + "dlls": [_logical(ctx, dll) for dll in target[VoiceExecutablePackageInfo].dlls], + } + for name, target in {"helper": ctx.attr.helper, "unit": ctx.attr.unit}.items() + }, "dumpbin": _logical(ctx, dumpbin), "runtime": [_logical(ctx, f) for f in inventory], "status": _logical(ctx, ctx.info_file), @@ -43,8 +52,8 @@ windows_runtime_diagnostic_test = rule( implementation = _impl, test = True, attrs = { - "helper": attr.label(mandatory = True, executable = True, cfg = "target"), - "unit": attr.label(mandatory = True, executable = True, cfg = "target"), + "helper": attr.label(mandatory = True, executable = True, cfg = "target", providers = [VoiceExecutablePackageInfo]), + "unit": attr.label(mandatory = True, executable = True, cfg = "target", providers = [VoiceExecutablePackageInfo]), "_runtime": attr.label(default = "//third_party/voice:native_link_windows_x86_64"), "_tools": attr.label(default = "//third_party/voice:windows_tools_x86_64", cfg = "exec", providers = [WindowsBuildToolsInfo]), "_launcher_template": attr.label(default = "//third_party/voice:windows_runtime_diagnostic.bat.tpl", allow_single_file = True), diff --git a/third_party/voice/windows_runtime_diagnostic.py b/third_party/voice/windows_runtime_diagnostic.py index 61105aa7c3fc..f126353c6695 100644 --- a/third_party/voice/windows_runtime_diagnostic.py +++ b/third_party/voice/windows_runtime_diagnostic.py @@ -137,6 +137,65 @@ def fixture(root, helper, runtime): return copied +def package_adjacency(executable, package, declared, env): + """Check exposed bytes and every declared DLL, without changing probe behavior.""" + raw = resolve(package["raw"], env) + receipt_path = resolve(package["receipt"], env) + if receipt_path.stat().st_size > 64 * 1024: + raise ValueError("packaging receipt size bound exceeded") + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + expected = { + name: path for name, path in declared.items() if path.suffix.lower() == ".dll" + } + if not expected: + raise ValueError("empty declared DLL closure") + supplied = inventory([resolve(key, env) for key in package["dlls"]]) + adjacent = inventory( + [ + path + for path in executable.parent.iterdir() + if path.is_file() and path.suffix.lower() == ".dll" + ] + ) + if len(adjacent) > 128: + raise ValueError("package adjacency inventory bound exceeded") + if supplied.keys() != expected.keys() or adjacent.keys() != expected.keys(): + raise ValueError("exposed executable lacks exact declared DLL siblings") + raw_sha, exposed_sha = digest(raw), digest(executable) + if raw_sha != exposed_sha: + raise ValueError("raw and exposed executable bytes differ") + if receipt["executable"] != { + "name": executable.name, + "source_sha256": raw_sha, + "output_sha256": exposed_sha, + }: + raise ValueError("executable packaging receipt mismatch") + recorded = {row["name"].lower(): row for row in receipt["dlls"]} + if len(recorded) != len(receipt["dlls"]) or recorded.keys() != expected.keys(): + raise ValueError("DLL packaging receipt inventory mismatch") + result = {} + for name, original in expected.items(): + copied = supplied[name] + expected_sha = digest(original) + if copied.parent != executable.parent or copied != adjacent[name]: + raise ValueError("declared packaged DLL is not executable-adjacent") + if digest(copied) != expected_sha or recorded[name] != { + "name": copied.name, + "source_sha256": expected_sha, + "output_sha256": expected_sha, + }: + raise ValueError("DLL packaging byte checksum mismatch") + result[name] = {"sha256": expected_sha, "matches_declared": True} + return { + "raw_path": str(raw), + "raw_sha256": raw_sha, + "exposed_sha256": exposed_sha, + "declared_dll_count": len(expected), + "dlls": result, + "valid": True, + } + + def diagnose(config, env, output): helper, unit, dumpbin, status = [ resolve(config[key], env) for key in ("helper", "unit", "dumpbin", "status") @@ -230,13 +289,26 @@ def diagnose(config, env, output): output["pe"].append( dict(parsed, path=str(path), edges=edges, inspection=result) ) - return all( - p.get("code") == 0 - and not p.get("timeout") - and not p.get("truncated") - and not p.get("reader_incomplete") - for p in output["probes"] - ) and all(p["stdout"].strip() == commit[0] for p in output["probes"][:2]) + # Keep original startup/PE metrics even when a package layout check fails. + output["packaging"] = {} + for name, executable in (("helper", helper), ("unit", unit)): + try: + output["packaging"][name] = package_adjacency( + executable, config["packages"][name], declared, env + ) + except (KeyError, ValueError, OSError, TypeError) as error: + output["packaging"][name] = {"valid": False, "error": str(error)} + return ( + all(value["valid"] for value in output["packaging"].values()) + and all( + p.get("code") == 0 + and not p.get("timeout") + and not p.get("truncated") + and not p.get("reader_incomplete") + for p in output["probes"] + ) + and all(p["stdout"].strip() == commit[0] for p in output["probes"][:2]) + ) def main(): From d59957da0766fdd3ae70401f9f62b7ab2473af2a Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 03:36:19 -0400 Subject: [PATCH 11/41] Bound observer pipes and fence owned process identity (refs #67) --- .../scripts/observe_windows_ci_resources.ps1 | 18 +- .../scripts/observe_windows_ci_resources.py | 320 ++++++++++++--- .../test_windows_ci_resource_observation.py | 381 +++++++++++++++--- .github/workflows/bazel.yml | 3 + 4 files changed, 606 insertions(+), 116 deletions(-) diff --git a/.github/scripts/observe_windows_ci_resources.ps1 b/.github/scripts/observe_windows_ci_resources.ps1 index dbf33066a27e..2b96cd7a5a72 100644 --- a/.github/scripts/observe_windows_ci_resources.ps1 +++ b/.github/scripts/observe_windows_ci_resources.ps1 @@ -1,5 +1,5 @@ # Numeric-only collector. Called by the bounded supervisor, never directly by CI. -param([int]$OwnedPid = 0, [double]$BuildStart = 0) +param([int]$OwnedPid = 0, [long]$CreationMicroseconds = 0) $ErrorActionPreference = 'Stop' $result = [ordered]@{ unavailable = 0; truncated = $false; memory = $null; pagefiles = @(); configured_pagefiles = $null; volumes = @(); processes = @(); process_scope = 'live-owned-descendant-snapshot-only-detached-servers-unobserved'; vhd = $null } function Number($Value, [long]$Scale = 1) { if ($null -eq $Value) { return $null }; return [long]$Value * $Scale } @@ -53,9 +53,13 @@ if ($OwnedPid -gt 0) { $queue = [Collections.Generic.Queue[int]]::new() $seen = [Collections.Generic.HashSet[int]]::new() try { + if ($CreationMicroseconds -le 0) { throw 'ownership unavailable' } $root = Query Win32_Process "ProcessId=$OwnedPid" - $created = ([DateTimeOffset]$root.CreationDate).ToUnixTimeMilliseconds() / 1000.0 - if (-not $root -or [Math]::Abs($created - $BuildStart) -gt 2) { throw 'ownership unavailable' } + # CIM DateTime precision is microseconds; match GetProcessTimes FILETIME + # truncated from 100 ns to microseconds since the 1601 UTC epoch. + $epochTicks = [datetime]::new(1601, 1, 1, 0, 0, 0, [DateTimeKind]::Utc).Ticks + $created = [long][Math]::Floor(($root.CreationDate.ToUniversalTime().Ticks - $epochTicks) / 10D) + if (-not $root -or $created -ne $CreationMicroseconds) { throw 'ownership unavailable' } } catch { $result.unavailable++; $result | ConvertTo-Json -Depth 5 -Compress; exit 0 } $creation = @{ $OwnedPid = $root.CreationDate } $queue.Enqueue($OwnedPid) @@ -65,10 +69,14 @@ if ($OwnedPid -gt 0) { try { $current = Query Win32_Process "ProcessId=$parent" if (-not $current -or $current.CreationDate -ne $creation[$parent]) { $result.unavailable++; continue } - foreach ($p in @(Query Win32_Process "ParentProcessId=$parent")) { + $children = @(Query Win32_Process "ParentProcessId=$parent") + # Fence PID reuse during the child query as well as before it. + $after = Query Win32_Process "ProcessId=$parent" + if (-not $after -or $after.CreationDate -ne $creation[$parent]) { $result.unavailable++; continue } + foreach ($p in $children) { if ($seen.Count + $queue.Count -ge 128) { $result.truncated = $true; break } # A reused PID older than this build cannot be an owned child. - if (([DateTimeOffset]$p.CreationDate).ToUnixTimeMilliseconds() / 1000.0 -lt $created) { $result.unavailable++; continue } + if ($p.CreationDate -lt $root.CreationDate) { $result.unavailable++; continue } $creation[[int]$p.ProcessId] = $p.CreationDate $queue.Enqueue([int]$p.ProcessId) if ($p.Name -match '^(rustc|link|cl|ninja|cmake|python|python3|bazel|bash)(\.exe)?$') { diff --git a/.github/scripts/observe_windows_ci_resources.py b/.github/scripts/observe_windows_ci_resources.py index d4c23eebb2a2..892a4a9bf1b2 100644 --- a/.github/scripts/observe_windows_ci_resources.py +++ b/.github/scripts/observe_windows_ci_resources.py @@ -1,11 +1,16 @@ """Job-local, bounded observation; never changes resource budgets or host state.""" + +import ctypes import json +import math +import re import os from pathlib import Path import platform import subprocess import sys import time +import threading MAX_SAMPLE = 16 * 1024 MAX_TOTAL = 2 * 1024 * 1024 @@ -15,38 +20,221 @@ def authorized(env=None, system=None): env = os.environ if env is None else env - return (env.get('GITHUB_REPOSITORY') == 'CompleteDotTech/codex-subagents-multi-provider' - and env.get('GITHUB_ACTIONS') == 'true' - and env.get('RUNNER_OS') == 'Windows' - and env.get('RUNNER_ENVIRONMENT') == 'github-hosted' - and (platform.system() if system is None else system) == 'Windows') + return ( + env.get("GITHUB_REPOSITORY") == "CompleteDotTech/codex-subagents-multi-provider" + and env.get("GITHUB_ACTIONS") == "true" + and env.get("RUNNER_OS") == "Windows" + and env.get("RUNNER_ENVIRONMENT") == "github-hosted" + and (platform.system() if system is None else system) == "Windows" + ) def identity(): def git(ref): try: - value = subprocess.check_output(['git', 'rev-parse', ref], timeout=2, - stderr=subprocess.DEVNULL, text=True).strip() - return value if len(value) == 40 and all(c in '0123456789abcdef' for c in value) else None + value = subprocess.check_output( + ["git", "rev-parse", ref], + timeout=2, + stderr=subprocess.DEVNULL, + text=True, + ).strip() + return ( + value + if len(value) == 40 and all(c in "0123456789abcdef" for c in value) + else None + ) except (OSError, subprocess.SubprocessError): return None - return dict(head=git('HEAD'), tree=git('HEAD^{tree}'), - run=os.environ.get('GITHUB_RUN_ID'), attempt=os.environ.get('GITHUB_RUN_ATTEMPT'), - job=os.environ.get('GITHUB_JOB'), architecture=platform.machine()) + + return dict( + head=git("HEAD"), + tree=git("HEAD^{tree}"), + run=os.environ.get("GITHUB_RUN_ID"), + attempt=os.environ.get("GITHUB_RUN_ATTEMPT"), + job=os.environ.get("GITHUB_JOB"), + architecture=platform.machine(), + ) + + +def bounded_output(command, timeout=10): + """Read at most MAX_SAMPLE bytes; discard stderr, kill/reap only our collector. + + The collector script starts no subprocesses. Its owned stdout reader therefore + reaches EOF when the collector exits or is killed, and is joined before return. + """ + child = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + deadline = time.monotonic() + timeout + data = bytearray() + done = threading.Event() + overflow = threading.Event() + failed = threading.Event() + + def read(): + try: + while True: + chunk = child.stdout.read(1024) + if not chunk: + break + remaining = MAX_SAMPLE - len(data) + data.extend(chunk[:remaining]) + if len(chunk) > remaining: + overflow.set() + break + except OSError: + failed.set() + finally: + done.set() + + reader = threading.Thread(target=read, name="windows-resource-collector-reader") + try: + reader.start() + if not done.wait(max(0, deadline - time.monotonic())): + raise subprocess.TimeoutExpired("resource collector", timeout) + if overflow.is_set(): + raise OverflowError("resource collector limit") + if failed.is_set(): + raise OSError("resource collector read") + child.wait(timeout=max(0.001, deadline - time.monotonic())) + if child.returncode: + raise ValueError("resource collector failed") + return bytes(data) + finally: + if child.poll() is None: + child.kill() + child.wait() + if reader.ident is not None: + reader.join() + child.stdout.close() + + +def owned_creation_microseconds(child, api=None): + """GetProcessTimes on Popen's OWNED handle, microseconds since 1601 UTC. + + CIM CreationDate has microsecond precision; truncate the Windows FILETIME's + 100 ns fraction once. Do not infer identity from PID or launch wall clocks. + """ + try: + if api is None: + if platform.system() != "Windows": + return None + api = ctypes.WinDLL("kernel32", use_last_error=True).GetProcessTimes + api.argtypes = [ctypes.c_void_p] * 5 + api.restype = ctypes.c_int + values = [ctypes.c_uint64() for _ in range(4)] + if not api(int(child._handle), *(ctypes.byref(value) for value in values)): + return None + return values[0].value // 10 + except (AttributeError, OSError, TypeError, ValueError): + return None + + +def sanitize(data): + """Only a fixed numeric schema and fixed labels may reach logs/artifacts.""" + if not isinstance(data, dict): + raise ValueError("invalid collector schema") + + def number(value): + return ( + value + if type(value) in (int, float) and math.isfinite(value) and value >= 0 + else None + ) + + def fields(row, names): + return {name: number(row.get(name)) for name in names} + + def volume(value): + return ( + value.rstrip("\\").upper() + if isinstance(value, str) and re.fullmatch(r"[A-Za-z]:\\?", value) + else None + ) + + result = dict( + unavailable=number(data.get("unavailable")) or 0, + truncated=data.get("truncated") is True, + process_scope="live-owned-descendant-snapshot-only-detached-servers-unobserved", + ) + memory = data.get("memory") + result["memory"] = ( + fields( + memory, + ( + "available_bytes", + "committed_bytes", + "commit_limit_bytes", + "commit_peak_bytes", + "percent_commit", + ), + ) + if isinstance(memory, dict) + else None + ) + tables = { + "volumes": (16, ("free_bytes", "total_bytes")), + "pagefiles": (16, ("allocated_bytes", "current_bytes", "peak_bytes")), + "configured_pagefiles": (16, ("initial_bytes", "maximum_bytes")), + "processes": (128, ("working_bytes", "private_bytes")), + } + for name, (limit, names) in tables.items(): + rows = data.get(name) + if not isinstance(rows, list): + result[name] = None + continue + result[name] = [] + if len(rows) > limit: + result["truncated"] = True + for row in rows[:limit]: + if not isinstance(row, dict): + result["unavailable"] += 1 + continue + cleaned = fields(row, names) + if name == "processes": + image = row.get("image") + if not isinstance(image, str) or not re.fullmatch( + r"(rustc|link|cl|ninja|cmake|python|python3|bazel|bash)(\.exe)?", + image, + ): + result["unavailable"] += 1 + continue + cleaned["image"] = image + else: + cleaned["volume"] = volume(row.get("volume")) + if cleaned["volume"] is None: + result["unavailable"] += 1 + result[name].append(cleaned) + vhd = data.get("vhd") + result["vhd"] = None + if isinstance(vhd, dict): + result["vhd"] = fields( + vhd, ("file_length_bytes", "logical_size_bytes", "allocated_physical_bytes") + ) + result["vhd"].update( + backing_volume=volume(vhd.get("backing_volume")), + logical_size_source="setup-dev-drive.ps1", + ) + return result -def collect(owned_pid=0, start=0): +def collect(owned_pid=0, start=None): try: - completed = subprocess.run(['pwsh', '-NoProfile', '-NonInteractive', '-File', - str(Path(__file__).with_suffix('.ps1')), - '-OwnedPid', str(owned_pid), '-BuildStart', str(start)], timeout=10, - capture_output=True, text=True, check=True) - if len(completed.stdout.encode('utf-8')) > MAX_SAMPLE: - return dict(unavailable=1, truncated=True) - data = json.loads(completed.stdout) - if not isinstance(data, dict): - raise ValueError('invalid result') - return data + output = bounded_output( + [ + "pwsh", + "-NoProfile", + "-NonInteractive", + "-File", + str(Path(__file__).with_suffix(".ps1")), + "-OwnedPid", + str(owned_pid), + "-CreationMicroseconds", + str(start or 0), + ], + timeout=10, + ) + return sanitize(json.loads(output)) + except OverflowError: + return dict(unavailable=1, truncated=True) except (OSError, subprocess.SubprocessError, ValueError): # Do not print exceptions: they may contain command or environment text. return dict(unavailable=1, truncated=False) @@ -55,14 +243,18 @@ def collect(owned_pid=0, start=0): class Observer: def __init__(self): self.provenance = identity() - self.path = Path(os.environ['RUNNER_TEMP']) / 'voice-runtime-diagnostic' / 'resources.jsonl' + self.path = ( + Path(os.environ["RUNNER_TEMP"]) + / "voice-runtime-diagnostic" + / "resources.jsonl" + ) self.count = 0 self.size = 0 self.exhausted = False try: self.size = self.path.stat().st_size if self.size <= MAX_TOTAL: - with self.path.open('rb') as source: + with self.path.open("rb") as source: self.count = sum(1 for _ in source) else: self.exhausted = True @@ -73,35 +265,56 @@ def __init__(self): def sample(self, phase, owned_pid=0, start=0): if self.exhausted or self.count >= MAX_SAMPLES or self.size >= MAX_TOTAL: - print('windows-resources exhausted=1', flush=True) + print("windows-resources exhausted=1", flush=True) return - data = collect(owned_pid, start) - record = dict(identity=self.provenance, phase=phase, unix_seconds=int(time.time()), **data) - line = json.dumps(record, separators=(',', ':')).encode('utf-8') + b'\n' + data = sanitize(collect(owned_pid, start)) + record = dict( + identity=self.provenance, phase=phase, unix_seconds=int(time.time()), **data + ) + line = json.dumps(record, separators=(",", ":")).encode("utf-8") + b"\n" if len(line) > MAX_SAMPLE: - record = dict(identity=self.provenance, phase=phase, unix_seconds=int(time.time()), - unavailable=1, truncated=True) - line = json.dumps(record, separators=(',', ':')).encode('utf-8') + b'\n' + record = dict( + identity=self.provenance, + phase=phase, + unix_seconds=int(time.time()), + unavailable=1, + truncated=True, + ) + line = json.dumps(record, separators=(",", ":")).encode("utf-8") + b"\n" # Console precedes disk I/O, including when the runner disk is full. - memory = data.get('memory') or {} - numeric = {key: value for key, value in memory.items() - if value is None or type(value) in (int, float)} - print('windows-resources ' + json.dumps(dict(phase=phase, memory=numeric, - volumes=data.get('volumes', []), pagefiles=data.get('pagefiles', []), - unavailable=data.get('unavailable', 0), truncated=data.get('truncated', False)), - separators=(',', ':'))[:1900], flush=True) + memory = data.get("memory") or {} + numeric = { + key: value + for key, value in memory.items() + if value is None or type(value) in (int, float) + } + print( + "windows-resources " + + json.dumps( + dict( + phase=phase, + memory=numeric, + volumes=data.get("volumes", []), + pagefiles=data.get("pagefiles", []), + unavailable=data.get("unavailable", 0), + truncated=data.get("truncated", False), + ), + separators=(",", ":"), + )[:1900], + flush=True, + ) self.count += 1 if self.size + len(line) > MAX_TOTAL: self.exhausted = True return try: self.path.parent.mkdir(parents=True, exist_ok=True) - with self.path.open('ab') as target: + with self.path.open("ab") as target: target.write(line) target.flush() self.size += len(line) except OSError: - print('windows-resources artifact_unavailable=1', flush=True) + print("windows-resources artifact_unavailable=1", flush=True) # Keep bounded console observations even if disk writes fail. self.size += len(line) @@ -111,14 +324,14 @@ def safe_sample(observer, phase, pid=0, start=0): try: observer.sample(phase, pid, start) except Exception: - print('windows-resources unavailable=1', flush=True) + print("windows-resources unavailable=1", flush=True) def supervise(command, observer): - safe_sample(observer, 'before-build') + safe_sample(observer, "before-build") # Argument vector, inherited console, no shell interpolation or output rewriting. - start = time.time() child = subprocess.Popen(command) + start = owned_creation_microseconds(child) deadline = time.monotonic() + INTERVAL try: while True: @@ -126,12 +339,12 @@ def supervise(command, observer): code = child.wait(timeout=max(0, deadline - time.monotonic())) break except subprocess.TimeoutExpired: - safe_sample(observer, 'build', child.pid, start) + safe_sample(observer, "build", child.pid, start) deadline += INTERVAL finally: # No detached sampler exists. Collector subprocesses are synchronously - # reaped (and on timeout killed) by subprocess.run. Never kill host tools. - safe_sample(observer, 'build-exit') + # reaped (and on timeout killed) by bounded_output. Never kill host tools. + safe_sample(observer, "build-exit") return code @@ -141,14 +354,19 @@ def main(args): try: observer = Observer() except Exception: - print('windows-resources unavailable=1', flush=True) - if args[:2] == ['supervise', '--'] and len(args) > 2: + print("windows-resources unavailable=1", flush=True) + if args[:2] == ["supervise", "--"] and len(args) > 2: return supervise(args[2:], observer) - if len(args) == 1 and args[0] in ('before-prepare', 'after-provision', 'after-prepare', 'after-artifacts'): + if len(args) == 1 and args[0] in ( + "before-prepare", + "after-provision", + "after-prepare", + "after-artifacts", + ): safe_sample(observer, args[0]) return 0 - raise SystemExit('expected phase or supervise -- command arguments') + raise SystemExit("expected phase or supervise -- command arguments") -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main(sys.argv[1:])) diff --git a/.github/scripts/test_windows_ci_resource_observation.py b/.github/scripts/test_windows_ci_resource_observation.py index 72217c42fcf2..f9822641b1ce 100644 --- a/.github/scripts/test_windows_ci_resource_observation.py +++ b/.github/scripts/test_windows_ci_resource_observation.py @@ -1,4 +1,5 @@ """Fixture-only checks: no host CIM queries or actual builds.""" + import contextlib import importlib.util import io @@ -6,102 +7,356 @@ from pathlib import Path import subprocess import tempfile +import sys +import threading +import time import unittest from unittest.mock import patch, Mock -SPEC = importlib.util.spec_from_file_location('observation', Path(__file__).with_name('observe_windows_ci_resources.py')) +SPEC = importlib.util.spec_from_file_location( + "observation", Path(__file__).with_name("observe_windows_ci_resources.py") +) m = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(m) class ObservationTests(unittest.TestCase): def test_scope_requires_every_gate_and_actual_windows(self): - env = dict(GITHUB_REPOSITORY='CompleteDotTech/codex-subagents-multi-provider', GITHUB_ACTIONS='true', RUNNER_OS='Windows', RUNNER_ENVIRONMENT='github-hosted') - self.assertTrue(m.authorized(env, 'Windows')) + env = dict( + GITHUB_REPOSITORY="CompleteDotTech/codex-subagents-multi-provider", + GITHUB_ACTIONS="true", + RUNNER_OS="Windows", + RUNNER_ENVIRONMENT="github-hosted", + ) + self.assertTrue(m.authorized(env, "Windows")) for key in env: - altered = dict(env, **{key: 'other'}) - self.assertFalse(m.authorized(altered, 'Windows')) - self.assertFalse(m.authorized(env, 'Linux')) + altered = dict(env, **{key: "other"}) + self.assertFalse(m.authorized(altered, "Windows")) + self.assertFalse(m.authorized(env, "Linux")) def test_timeout_and_errors_do_not_leak(self): - for error in (OSError('SECRET'), subprocess.TimeoutExpired('SECRET', 10), ValueError('SECRET')): - with patch.object(m.subprocess, 'run', side_effect=error): + for error in ( + OSError("SECRET"), + subprocess.TimeoutExpired("SECRET", 10), + ValueError("SECRET"), + ): + with patch.object(m, "bounded_output", side_effect=error): result = m.collect(123, 1) - self.assertEqual(result['unavailable'], 1) - self.assertNotIn('SECRET', json.dumps(result)) + self.assertEqual(result["unavailable"], 1) + self.assertNotIn("SECRET", json.dumps(result)) def test_collection_deadline_and_no_shell(self): - with patch.object(m.subprocess, 'run', return_value=Mock(stdout='{}')) as run: + with patch.object(m, "bounded_output", return_value=b"{}") as run: m.collect(123, 4) - self.assertEqual(run.call_args.kwargs['timeout'], 10) - self.assertNotIn('shell', run.call_args.kwargs) - self.assertEqual(run.call_args.args[0][-4:], ['-OwnedPid', '123', '-BuildStart', '4']) + self.assertEqual(run.call_args.kwargs["timeout"], 10) + self.assertNotIn("shell", run.call_args.kwargs) + self.assertEqual( + run.call_args.args[0][-4:], + ["-OwnedPid", "123", "-CreationMicroseconds", "4"], + ) def test_build_exit_preserved_with_observer_failure(self): observer = Mock() - observer.sample.side_effect = OSError('SECRET') + observer.sample.side_effect = OSError("SECRET") child = Mock() - child.wait.side_effect = [subprocess.TimeoutExpired('fixture', 60), 37] + child.wait.side_effect = [subprocess.TimeoutExpired("fixture", 60), 37] child.pid = 123 - with patch.object(m.subprocess, 'Popen', return_value=child) as launch, contextlib.redirect_stdout(io.StringIO()) as out: - self.assertEqual(m.supervise(['bash', 'file with spaces', '--jobs=8'], observer), 37) - self.assertNotIn('SECRET', out.getvalue()) - launch.assert_called_once_with(['bash', 'file with spaces', '--jobs=8']) + with ( + patch.object(m.subprocess, "Popen", return_value=child) as launch, + contextlib.redirect_stdout(io.StringIO()) as out, + ): + self.assertEqual( + m.supervise(["bash", "file with spaces", "--jobs=8"], observer), 37 + ) + self.assertNotIn("SECRET", out.getvalue()) + launch.assert_called_once_with(["bash", "file with spaces", "--jobs=8"]) child.kill.assert_not_called() - self.assertEqual(observer.sample.call_args_list[-1].args[0], 'build-exit') + self.assertEqual(observer.sample.call_args_list[-1].args[0], "build-exit") def test_start_failure_is_real_and_not_success(self): - with patch.object(m.subprocess, 'Popen', side_effect=FileNotFoundError('fixture')): + with patch.object( + m.subprocess, "Popen", side_effect=FileNotFoundError("fixture") + ): with self.assertRaises(FileNotFoundError): - m.supervise(['missing-build'], None) + m.supervise(["missing-build"], None) def test_oversized_collector_rejected(self): - with patch.object(m.subprocess, 'run', return_value=Mock(stdout='x' * (m.MAX_SAMPLE + 1))): - self.assertEqual(m.collect(), {'unavailable': 1, 'truncated': True}) + with patch.object(m, "bounded_output", side_effect=OverflowError()): + self.assertEqual(m.collect(), {"unavailable": 1, "truncated": True}) + + def test_real_collector_overflow_is_bounded_and_reader_reaped(self): + started = time.monotonic() + with self.assertRaises(OverflowError): + m.bounded_output( + [ + sys.executable, + "-c", + "import os,time; os.write(1,b'x'*1000000); time.sleep(30)", + ], + timeout=2, + ) + self.assertLess(time.monotonic() - started, 3) + self.assertFalse( + any( + t.name == "windows-resource-collector-reader" + for t in threading.enumerate() + ) + ) + + def test_real_collector_timeout_and_stderr_are_bounded(self): + started = time.monotonic() + with self.assertRaises(subprocess.TimeoutExpired): + m.bounded_output( + [ + sys.executable, + "-c", + "import os,time; os.write(2,b'SECRET'*100000); time.sleep(30)", + ], + timeout=0.2, + ) + self.assertLess(time.monotonic() - started, 2) + self.assertFalse( + any( + t.name == "windows-resource-collector-reader" + for t in threading.enumerate() + ) + ) + + def test_real_collector_success_and_nonzero_cleanup(self): + self.assertEqual( + m.bounded_output([sys.executable, "-c", "print('{}')"]), + b"{}\r\n" if sys.platform == "win32" else b"{}\n", + ) + with self.assertRaises(ValueError): + m.bounded_output([sys.executable, "-c", "raise SystemExit(4)"]) + self.assertFalse( + any( + t.name == "windows-resource-collector-reader" + for t in threading.enumerate() + ) + ) + + def test_owned_process_handle_identity_has_microsecond_precision(self): + child = Mock(_handle=42) + + def api(handle, creation, exit_time, kernel, user): + self.assertEqual(handle, 42) + creation._obj.value = 123456789012345678 + return 1 + + self.assertEqual(m.owned_creation_microseconds(child, api), 12345678901234567) + self.assertIsNone(m.owned_creation_microseconds(child, lambda *args: 0)) + + def test_schema_rejects_secrets_nonfinite_and_nonallowlisted_images(self): + data = m.sanitize( + dict( + memory={"available_bytes": float("inf"), "SECRET": 1}, + volumes=[dict(volume="SECRET", free_bytes="SECRET", total_bytes=12)], + pagefiles=[dict(volume="C:\\", allocated_bytes=1048576)], + processes=[dict(image="SECRET", private_bytes=1)], + SECRET="SECRET", + vhd=dict(backing_volume="C:\\", logical_size_bytes=68719476736), + ) + ) + self.assertNotIn("SECRET", json.dumps(data)) + self.assertIsNone(data["memory"]["available_bytes"]) + self.assertIsNone(data["volumes"][0]["volume"]) + self.assertEqual(data["pagefiles"][0]["volume"], "C:") + self.assertEqual(data["processes"], []) + self.assertEqual(data["vhd"]["logical_size_bytes"], 68719476736) + self.assertIsNone(data["vhd"]["allocated_physical_bytes"]) def test_caps_and_stdout_precedes_write(self): - with tempfile.TemporaryDirectory() as temp, patch.dict(m.os.environ, {'RUNNER_TEMP': temp}), patch.object(m, 'identity', return_value={'head': 'fixture'}), patch.object(m, 'collect', return_value={'memory': {'available_bytes': 8}, 'unavailable': 0}): + with ( + tempfile.TemporaryDirectory() as temp, + patch.dict(m.os.environ, {"RUNNER_TEMP": temp}), + patch.object(m, "identity", return_value={"head": "fixture"}), + patch.object( + m, + "collect", + return_value={"memory": {"available_bytes": 8}, "unavailable": 0}, + ), + ): observer = m.Observer() with contextlib.redirect_stdout(io.StringIO()) as out: - observer.sample('before-prepare') - self.assertIn('available_bytes', out.getvalue()) + observer.sample("before-prepare") + self.assertIn("available_bytes", out.getvalue()) self.assertLess(observer.path.stat().st_size, m.MAX_SAMPLE) observer.count = m.MAX_SAMPLES with contextlib.redirect_stdout(io.StringIO()): - observer.sample('after-prepare') + observer.sample("after-prepare") self.assertEqual(len(observer.path.read_text().splitlines()), 1) observer.count = 0 observer.size = m.MAX_TOTAL with contextlib.redirect_stdout(io.StringIO()): - observer.sample('after-prepare') + observer.sample("after-prepare") self.assertEqual(len(observer.path.read_text().splitlines()), 1) def test_workflow_only_diagnostic_job_uses_observer(self): - workflow = Path(__file__).parents[1] / 'workflows' / 'bazel.yml' - before, job = workflow.read_text().split(' voice-runtime-diagnostic:', 1) - self.assertNotIn('observe_windows_ci_resources', before) - self.assertIn('python .github/scripts/observe_windows_ci_resources.py supervise -- bash ./.github/scripts/run-bazel-ci.sh', job) - self.assertIn('if: github.event_name == \'workflow_dispatch\' && inputs.voice_runtime_diagnostic', job) - self.assertEqual(job.count('if: always() && !cancelled()'), 4) - self.assertNotIn('--jobs', job) + workflow = Path(__file__).parents[1] / "workflows" / "bazel.yml" + before, job = workflow.read_text().split(" voice-runtime-diagnostic:", 1) + self.assertNotIn("observe_windows_ci_resources", before) + self.assertIn( + "python .github/scripts/observe_windows_ci_resources.py supervise -- bash ./.github/scripts/run-bazel-ci.sh", + job, + ) + self.assertIn( + "if: github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic", + job, + ) + self.assertEqual(job.count("if: always() && !cancelled()"), 4) + self.assertNotIn("--jobs", job) + for step in job.split(" - name: ")[1:]: + if step.startswith("Observe resources "): + self.assertIn("continue-on-error: true", step) def test_nested_observation_input_and_order(self): - actions = Path(__file__).parents[1] / 'actions' - prepare = (actions / 'prepare-bazel-ci' / 'action.yml').read_text() - setup = (actions / 'setup-bazel-ci' / 'action.yml').read_text() + actions = Path(__file__).parents[1] / "actions" + prepare = (actions / "prepare-bazel-ci" / "action.yml").read_text() + setup = (actions / "setup-bazel-ci" / "action.yml").read_text() for text in (prepare, setup): - self.assertIn('windows-resource-observation:', text) - self.assertIn('default: "false"', text.split('windows-resource-observation:', 1)[1]) - self.assertIn('windows-resource-observation: ${{ inputs.windows-resource-observation }}', prepare) - self.assertLess(setup.index('id: setup_ci'), setup.index('after-provision')) - self.assertLess(setup.index('after-provision'), setup.index('Provision pinned native Windows')) - workflow = (Path(__file__).parents[1] / 'workflows' / 'bazel.yml').read_text() - self.assertEqual(workflow.count('windows-resource-observation: true'), 1) - self.assertNotIn('windows-resource-observation', workflow.split(' voice-runtime-diagnostic:')[0]) + self.assertIn("windows-resource-observation:", text) + self.assertIn( + 'default: "false"', text.split("windows-resource-observation:", 1)[1] + ) + self.assertIn( + "windows-resource-observation: ${{ inputs.windows-resource-observation }}", + prepare, + ) + self.assertLess(setup.index("id: setup_ci"), setup.index("after-provision")) + self.assertLess( + setup.index("after-provision"), + setup.index("Provision pinned native Windows"), + ) + workflow = (Path(__file__).parents[1] / "workflows" / "bazel.yml").read_text() + self.assertEqual(workflow.count("windows-resource-observation: true"), 1) + self.assertNotIn( + "windows-resource-observation", + workflow.split(" voice-runtime-diagnostic:")[0], + ) + + def run_ownership_fixture( + self, offset=0, children=2, missing_identity=False, reuse_during_query=False + ): + script = Path(__file__).with_name("observe_windows_ci_resources.ps1").resolve() + fixture = r""" +$global:fixtureQueries = 0 +$global:fixtureCreated = [datetime]::new(2026, 1, 1, 0, 0, 0, [DateTimeKind]::Utc) +function global:Get-CimInstance { + param($ClassName, $Filter, $OperationTimeoutSec, $ErrorAction) + switch ($ClassName) { + 'Win32_PerfFormattedData_PerfOS_Memory' { [pscustomobject]@{ AvailableBytes=$null; CommittedBytes=200; CommitLimit=400; PercentCommittedBytesInUse=50 } } + 'Win32_PageFileUsage' { [pscustomobject]@{ Name='C:\pagefile.sys'; AllocatedBaseSize=$null; CurrentUsage=1; PeakUsage=2 } } + 'Win32_PageFileSetting' { [pscustomobject]@{ Name='C:\pagefile.sys'; InitialSize=2; MaximumSize=4 } } + 'Win32_LogicalDisk' { [pscustomobject]@{ FreeSpace=1024; Size=8192 } } + 'Win32_Process' { + if ($Filter -eq 'ProcessId=101') { $global:fixtureQueries++; $date=$global:fixtureCreated; if (REUSE -and $global:fixtureQueries -ge 3) { $date=$date.AddTicks(10) }; [pscustomobject]@{ ProcessId=101; CreationDate=$date }; return } + if ($Filter -eq 'ParentProcessId=101') { + 1..CHILDREN | ForEach-Object { [pscustomobject]@{ ProcessId=(200+$_); CreationDate=$global:fixtureCreated.AddMilliseconds(1); Name='rustc.exe'; WorkingSetSize=100; PrivatePageCount=$null } }; return + } + if ($Filter -match '^ProcessId=\d+$') { [pscustomobject]@{ CreationDate=$global:fixtureCreated.AddMilliseconds(1) }; return } + if ($Filter -match '^ParentProcessId=\d+$') { return } + throw 'unscoped host query prohibited' + } + default { throw 'host query prohibited' } + } +} +function global:Test-Path { + param($LiteralPath, $PathType) + if ($LiteralPath -ne 'C:\fixture\codex-dev-drive.vhdx') { throw 'unexpected VHD path' } + return $true +} +function global:Get-Item { + param($LiteralPath) + if ($LiteralPath -ne 'C:\fixture\codex-dev-drive.vhdx') { throw 'unexpected VHD path' } + [pscustomobject]@{ Length=4096 } +} +$env:RUNNER_TEMP = 'C:\fixture' +$epoch = [datetime]::new(1601, 1, 1, 0, 0, 0, [DateTimeKind]::Utc).Ticks +$micro = [long](($global:fixtureCreated.Ticks - $epoch) / 10D) + OFFSET +& $args[0] -OwnedPid 101 -CreationMicroseconds $micro +""" + fixture = fixture.replace("CHILDREN", str(children)).replace( + "OFFSET", str(offset) + ) + fixture = fixture.replace("REUSE", "$true" if reuse_during_query else "$false") + if missing_identity: + fixture = fixture.replace( + "-CreationMicroseconds $micro", "-CreationMicroseconds 0" + ) + with tempfile.TemporaryDirectory() as temp: + runner = Path(temp) / "fixture.ps1" + runner.write_text(fixture) + result = subprocess.run( + ["pwsh", "-NoProfile", "-File", str(runner), str(script)], + capture_output=True, + text=True, + timeout=10, + check=True, + ) + return json.loads(result.stdout) + + def test_powershell_exact_root_identity_and_numeric_vhd_fixture(self): + data = self.run_ownership_fixture() + self.assertEqual(len(data["processes"]), 2) + self.assertEqual(data["processes"][0]["working_bytes"], 100) + self.assertIsNone(data["processes"][0]["private_bytes"]) + self.assertIsNone(data["memory"]) + self.assertIsNone(data["pagefiles"][0]["allocated_bytes"]) + self.assertEqual( + data["configured_pagefiles"][0]["maximum_bytes"], 4 * 1024 * 1024 + ) + self.assertEqual(data["vhd"]["file_length_bytes"], 4096) + self.assertEqual(data["vhd"]["logical_size_bytes"], 68719476736) + self.assertEqual(data["vhd"]["backing_volume"], "C:\\".replace("\\\\", "\\")) + self.assertIsNone(data["vhd"]["allocated_physical_bytes"]) + self.assertLessEqual(len(data["volumes"]), 16) + + def test_powershell_nearby_reused_root_and_missing_handle_rejected(self): + for options in ({"offset": 1000000}, {"offset": 1}, {"missing_identity": True}): + data = self.run_ownership_fixture(**options) + self.assertEqual(data["processes"], []) + self.assertGreater(data["unavailable"], 0) + + def test_powershell_parent_pid_reuse_during_query_rejected(self): + data = self.run_ownership_fixture(reuse_during_query=True) + self.assertEqual(data["processes"], []) + self.assertGreater(data["unavailable"], 0) + + def test_artifact_failure_does_not_remove_prior_numeric_console(self): + with ( + tempfile.TemporaryDirectory() as temp, + patch.dict(m.os.environ, {"RUNNER_TEMP": temp}), + patch.object(m, "identity", return_value={}), + patch.object( + m, "collect", return_value={"memory": {"available_bytes": 123}} + ), + ): + observer = m.Observer() + output = io.StringIO() + + def fail_write(*args, **kwargs): + self.assertIn('"available_bytes":123', output.getvalue()) + raise OSError("SECRET disk error") + + with ( + contextlib.redirect_stdout(output), + patch.object(Path, "mkdir", side_effect=fail_write), + ): + observer.sample("after-prepare") + self.assertIn("artifact_unavailable=1", output.getvalue()) + self.assertNotIn("SECRET", output.getvalue()) + + def test_powershell_owned_descendants_capped(self): + data = self.run_ownership_fixture(children=200) + self.assertTrue(data["truncated"]) + self.assertLessEqual(len(data["processes"]), 128) + self.assertEqual(len(data["processes"]), 127) + self.assertLess(len(json.dumps(data).encode()), m.MAX_SAMPLE) def test_powershell_numeric_fixture_without_host_queries(self): - script = Path(__file__).with_name('observe_windows_ci_resources.ps1').resolve() - fixture = r''' + script = Path(__file__).with_name("observe_windows_ci_resources.ps1").resolve() + fixture = r""" function global:Get-CimInstance { param($ClassName, $Filter, $OperationTimeoutSec, $ErrorAction) switch ($ClassName) { @@ -113,20 +368,26 @@ def test_powershell_numeric_fixture_without_host_queries(self): } $env:RUNNER_TEMP = '' & $args[0] -''' +""" with tempfile.TemporaryDirectory() as temp: - runner = Path(temp) / 'fixture.ps1' + runner = Path(temp) / "fixture.ps1" runner.write_text(fixture) - result = subprocess.run(['pwsh', '-NoProfile', '-File', str(runner), str(script)], capture_output=True, text=True, timeout=10, check=True) + result = subprocess.run( + ["pwsh", "-NoProfile", "-File", str(runner), str(script)], + capture_output=True, + text=True, + timeout=10, + check=True, + ) data = json.loads(result.stdout) - self.assertEqual(data['memory']['committed_bytes'], 200) - self.assertEqual(data['pagefiles'][0]['allocated_bytes'], 2 * 1024 * 1024) - self.assertEqual(len(data['pagefiles']), 16) - self.assertTrue(data['truncated']) - self.assertIsNone(data['volumes'][0]['free_bytes']) - self.assertNotIn('SECRET', result.stdout) + self.assertEqual(data["memory"]["committed_bytes"], 200) + self.assertEqual(data["pagefiles"][0]["allocated_bytes"], 2 * 1024 * 1024) + self.assertEqual(len(data["pagefiles"]), 16) + self.assertTrue(data["truncated"]) + self.assertIsNone(data["volumes"][0]["free_bytes"]) + self.assertNotIn("SECRET", result.stdout) self.assertLess(len(result.stdout.encode()), m.MAX_SAMPLE) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 97629a7fa3b8..b7160bded53b 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -626,6 +626,7 @@ jobs: "VOICE_DIAGNOSTIC_HEAD=$(git rev-parse HEAD)" >> $env:GITHUB_ENV "VOICE_DIAGNOSTIC_TREE=$(git rev-parse 'HEAD^{tree}')" >> $env:GITHUB_ENV - name: Observe resources before preparation + continue-on-error: true shell: pwsh run: python .github/scripts/observe_windows_ci_resources.py before-prepare - name: Prepare Bazel CI @@ -636,6 +637,7 @@ jobs: windows-resource-observation: true cache-scope: bazel-test - name: Observe resources after preparation and input installation + continue-on-error: true if: always() && !cancelled() shell: pwsh run: python .github/scripts/observe_windows_ci_resources.py after-prepare @@ -663,6 +665,7 @@ jobs: if [[ -f "$file" ]]; then cp "$file" "${RUNNER_TEMP}/voice-runtime-diagnostic/"; fi done - name: Observe resources after text artifact collection + continue-on-error: true if: always() && !cancelled() shell: pwsh run: python .github/scripts/observe_windows_ci_resources.py after-artifacts From b25c132875f60d0373f3bdae8ce5074d88cc2d1a Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 03:40:58 -0400 Subject: [PATCH 12/41] Fence observed children to current parent lifetime (refs #67) --- .../scripts/observe_windows_ci_resources.ps1 | 5 ++-- .../test_windows_ci_resource_observation.py | 28 ++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/scripts/observe_windows_ci_resources.ps1 b/.github/scripts/observe_windows_ci_resources.ps1 index 2b96cd7a5a72..1b75957d5661 100644 --- a/.github/scripts/observe_windows_ci_resources.ps1 +++ b/.github/scripts/observe_windows_ci_resources.ps1 @@ -75,8 +75,9 @@ if ($OwnedPid -gt 0) { if (-not $after -or $after.CreationDate -ne $creation[$parent]) { $result.unavailable++; continue } foreach ($p in $children) { if ($seen.Count + $queue.Count -ge 128) { $result.truncated = $true; break } - # A reused PID older than this build cannot be an owned child. - if ($p.CreationDate -lt $root.CreationDate) { $result.unavailable++; continue } + # A stale child can retain a reused nonroot parent PID. It must + # have been created during this exact current parent lifetime. + if ($p.CreationDate -lt $current.CreationDate) { $result.unavailable++; continue } $creation[[int]$p.ProcessId] = $p.CreationDate $queue.Enqueue([int]$p.ProcessId) if ($p.Name -match '^(rustc|link|cl|ninja|cmake|python|python3|bazel|bash)(\.exe)?$') { diff --git a/.github/scripts/test_windows_ci_resource_observation.py b/.github/scripts/test_windows_ci_resource_observation.py index f9822641b1ce..98783732db71 100644 --- a/.github/scripts/test_windows_ci_resource_observation.py +++ b/.github/scripts/test_windows_ci_resource_observation.py @@ -236,7 +236,12 @@ def test_nested_observation_input_and_order(self): ) def run_ownership_fixture( - self, offset=0, children=2, missing_identity=False, reuse_during_query=False + self, + offset=0, + children=2, + missing_identity=False, + reuse_during_query=False, + stale_nonroot=False, ): script = Path(__file__).with_name("observe_windows_ci_resources.ps1").resolve() fixture = r""" @@ -251,6 +256,16 @@ def run_ownership_fixture( 'Win32_LogicalDisk' { [pscustomobject]@{ FreeSpace=1024; Size=8192 } } 'Win32_Process' { if ($Filter -eq 'ProcessId=101') { $global:fixtureQueries++; $date=$global:fixtureCreated; if (REUSE -and $global:fixtureQueries -ge 3) { $date=$date.AddTicks(10) }; [pscustomobject]@{ ProcessId=101; CreationDate=$date }; return } + if (STALEMODE) { + if ($Filter -eq 'ParentProcessId=101') { [pscustomobject]@{ ProcessId=201; CreationDate=$global:fixtureCreated.AddMilliseconds(2); Name='cmake.exe'; WorkingSetSize=200; PrivatePageCount=200 }; return } + if ($Filter -eq 'ProcessId=201') { [pscustomobject]@{ CreationDate=$global:fixtureCreated.AddMilliseconds(2) }; return } + if ($Filter -eq 'ParentProcessId=201') { + [pscustomobject]@{ ProcessId=301; CreationDate=$global:fixtureCreated.AddMilliseconds(1); Name='rustc.exe'; WorkingSetSize=100; PrivatePageCount=100 } + [pscustomobject]@{ ProcessId=302; CreationDate=$global:fixtureCreated.AddMilliseconds(3); Name='link.exe'; WorkingSetSize=300; PrivatePageCount=300 } + return + } + if ($Filter -eq 'ProcessId=302') { [pscustomobject]@{ CreationDate=$global:fixtureCreated.AddMilliseconds(3) }; return } + } if ($Filter -eq 'ParentProcessId=101') { 1..CHILDREN | ForEach-Object { [pscustomobject]@{ ProcessId=(200+$_); CreationDate=$global:fixtureCreated.AddMilliseconds(1); Name='rustc.exe'; WorkingSetSize=100; PrivatePageCount=$null } }; return } @@ -280,6 +295,7 @@ def run_ownership_fixture( "OFFSET", str(offset) ) fixture = fixture.replace("REUSE", "$true" if reuse_during_query else "$false") + fixture = fixture.replace("STALEMODE", "$true" if stale_nonroot else "$false") if missing_identity: fixture = fixture.replace( "-CreationMicroseconds $micro", "-CreationMicroseconds 0" @@ -347,6 +363,16 @@ def fail_write(*args, **kwargs): self.assertIn("artifact_unavailable=1", output.getvalue()) self.assertNotIn("SECRET", output.getvalue()) + def test_powershell_nonroot_parent_rejects_older_stale_child(self): + data = self.run_ownership_fixture(stale_nonroot=True) + self.assertEqual( + [row["image"] for row in data["processes"]], ["cmake.exe", "link.exe"] + ) + self.assertEqual( + [row["working_bytes"] for row in data["processes"]], [200, 300] + ) + self.assertGreater(data["unavailable"], 0) + def test_powershell_owned_descendants_capped(self): data = self.run_ownership_fixture(children=200) self.assertTrue(data["truncated"]) From b39ee85306dabcafbdd560c7525711c2001f5bb6 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 03:47:18 -0400 Subject: [PATCH 13/41] test(voice): keep provider claims out of copy unit tests Remove the new source-string consumer check; configured Bazel analysis and hosted diagnostics remain required for provider and routing evidence. Preserve all behavior tests. Refs #62 --- .../voice/test_package_windows_executable.py | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/third_party/voice/test_package_windows_executable.py b/third_party/voice/test_package_windows_executable.py index f644363873c1..ac174df99e04 100644 --- a/third_party/voice/test_package_windows_executable.py +++ b/third_party/voice/test_package_windows_executable.py @@ -168,41 +168,5 @@ def corrupt(_source, output, _length): self.assertFalse(Path(self.config["receipt"]).exists()) -class ConsumerContractTests(unittest.TestCase): - def test_raw_compilation_and_packaged_execution_routes_remain_distinct(self): - root = Path(__file__).resolve().parents[2] - source = (root / "defs.bzl").read_text(encoding="utf-8") - self.assertIn("binary_windows_runtime = {}", source) - self.assertIn( - 'binary_kwargs = {"binary_name": binary} if runtime else {}', source - ) - self.assertIn('crate = ":" + raw_binary', source) - self.assertIn('"//conditions:default": ":" + raw_binary', source) - self.assertIn('"//conditions:default": exposed_unit', source) - self.assertIn("test_bin = exposed_unit", source) - self.assertIn( - 'cargo_env["CARGO_BIN_EXE_" + binary] = "$(rlocationpath :%s)" % binary', - source, - ) - voice = (root / "codex-rs/voice-host/BUILD.bazel").read_text(encoding="utf-8") - self.assertIn( - 'unit = ":codex-voice-host-bin-unit-tests-bin__windows-runtime"', voice - ) - windows = (root / ".github/workflows/rust-release-windows.yml").read_text( - encoding="utf-8" - ) - self.assertIn( - "cp bazel-bin/codex-rs/voice-host/codex-voice-host__windows-runtime/codex-voice-host.exe", - windows, - ) - macos = (root / ".github/workflows/rust-release.yml").read_text( - encoding="utf-8" - ) - self.assertIn( - 'cp bazel-bin/codex-rs/voice-host/codex-voice-host "$output/codex-voice-host"', - macos, - ) - - if __name__ == "__main__": unittest.main() From a2a6cbfd198fa82d67a742e01e79c8c48bd1a11e Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 09:55:43 -0400 Subject: [PATCH 14/41] ci: qualify voice unit and installed client runtime in diagnostic --- .github/workflows/bazel.yml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index b39691f99085..0066998e368a 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -640,19 +640,30 @@ jobs: unset BUILDBUDDY_API_KEY ./.github/scripts/run-bazel-ci.sh \ --print-failed-test-logs --print-failed-action-summary \ - --windows-cross-compile -- test --nocache_test_results --test_output=all --zip_undeclared_test_outputs=false \ + --windows-cross-compile -- test --keep_going --nocache_test_results --test_output=all --zip_undeclared_test_outputs=false \ --test_env=VOICE_DIAGNOSTIC_HEAD --test_env=VOICE_DIAGNOSTIC_TREE \ --test_env=GITHUB_SHA --test_env=GITHUB_EVENT_NAME --test_env=GITHUB_REF --test_env=GITHUB_RUN_ID \ --build_metadata=COMMIT_SHA=${GITHUB_SHA} -- \ - //codex-rs/voice-host:voice-host-windows-runtime-diagnostic-test + //codex-rs/voice-host:voice-host-windows-runtime-diagnostic-test \ + //codex-rs/voice-host:codex-voice-host-bin-unit-tests \ + //codex-rs/voice-host:voice-host-installed_client-test - name: Collect diagnostic text outputs if: always() && !cancelled() shell: bash run: | mkdir -p "${RUNNER_TEMP}/voice-runtime-diagnostic" - # Copy only the named text result/log, never executables or runfiles. - for file in bazel-testlogs/codex-rs/voice-host/voice-host-windows-runtime-diagnostic-test/test.outputs/voice-runtime-diagnostic.json bazel-testlogs/codex-rs/voice-host/voice-host-windows-runtime-diagnostic-test/test.log; do - if [[ -f "$file" ]]; then cp "$file" "${RUNNER_TEMP}/voice-runtime-diagnostic/"; fi + # Retain only named text evidence, never executables or runfiles. + result=bazel-testlogs/codex-rs/voice-host/voice-host-windows-runtime-diagnostic-test/test.outputs/voice-runtime-diagnostic.json + if [[ -f "$result" ]]; then cp "$result" "${RUNNER_TEMP}/voice-runtime-diagnostic/"; fi + for target in voice-host-windows-runtime-diagnostic-test codex-voice-host-bin-unit-tests voice-host-installed_client-test; do + file="bazel-testlogs/codex-rs/voice-host/${target}/test.log" + if [[ -f "$file" ]]; then + head -c 8388608 "$file" > "${RUNNER_TEMP}/voice-runtime-diagnostic/${target}.test.log" + if [[ $(wc -c < "$file") -gt 8388608 ]]; then + echo "${target}: retained first 8388608 bytes; original test.log was larger" + printf '%s\n' 'Retained first 8388608 bytes of test.log.' > "${RUNNER_TEMP}/voice-runtime-diagnostic/${target}.truncated.txt" + fi + fi done - name: Upload diagnostic text evidence if: always() && !cancelled() From 51739682dd9916431642f7f6c262d55158454209 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 09:59:30 -0400 Subject: [PATCH 15/41] test: require expanded voice diagnostic target contract --- .github/scripts/test_voice_runtime_diagnostic_workflow.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/scripts/test_voice_runtime_diagnostic_workflow.py b/.github/scripts/test_voice_runtime_diagnostic_workflow.py index 24e095553633..f40d7bfda3c0 100644 --- a/.github/scripts/test_voice_runtime_diagnostic_workflow.py +++ b/.github/scripts/test_voice_runtime_diagnostic_workflow.py @@ -61,9 +61,14 @@ def test_diagnostic_scope_and_credentials(self): self.assertIn("persist-credentials: false", job) self.assertIn("native-windows-voice: true", job) self.assertIn("--nocache_test_results", job) + self.assertIn("--keep_going", job) self.assertEqual( re.findall(r"//codex-rs/[^\s]+", job), - ["//codex-rs/voice-host:voice-host-windows-runtime-diagnostic-test"], + [ + "//codex-rs/voice-host:voice-host-windows-runtime-diagnostic-test", + "//codex-rs/voice-host:codex-voice-host-bin-unit-tests", + "//codex-rs/voice-host:voice-host-installed_client-test", + ], ) self.assertNotIn("//...", job) self.assertIn("'voice-diagnostic' || 'normal'", self.text.split("jobs:", 1)[0]) From 6a58bb8727370f0818b4e03f1d42333fdae068c5 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 10:07:47 -0400 Subject: [PATCH 16/41] ci: declare credential-free Windows diagnostic preparation --- .github/scripts/test_hosted_storage.py | 19 ++++++++++++++++++- .github/workflows/bazel.yml | 1 + 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/scripts/test_hosted_storage.py b/.github/scripts/test_hosted_storage.py index 46a9c699e031..ae86e8ce5cc7 100644 --- a/.github/scripts/test_hosted_storage.py +++ b/.github/scripts/test_hosted_storage.py @@ -472,9 +472,26 @@ def test_every_prepare_caller_explicitly_reports_credential_availability(self): content.count("uses: ./.github/actions/prepare-bazel-ci"), str(path), ) + diagnostic = ( + re.search( + r"^ voice-runtime-diagnostic:\n(.*?)(?=^ [\w-]+:|\Z)", + content, + re.M | re.S, + ) + if path == github / "workflows/bazel.yml" + else None + ) for call in calls: params = call.group(2) - if directory.name == "workflows": + if ( + diagnostic + and diagnostic.start() < call.start() < diagnostic.end() + ): + expected = "false" + # This isolated route never receives a BuildBuddy secret. + self.assertNotIn("secrets.", diagnostic.group(1)) + self.assertIn("unset BUILDBUDDY_API_KEY", diagnostic.group(1)) + elif directory.name == "workflows": expected = "${{ secrets.BUILDBUDDY_API_KEY != '' }}" elif path.parent.name == "run-argument-comment-lint": expected = "${{ inputs.buildbuddy-api-key != '' }}" diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index b34153cee151..ed77877d4cf2 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -715,6 +715,7 @@ jobs: target: x86_64-pc-windows-gnullvm native-windows-voice: true windows-resource-observation: true + remote-credentials: false cache-scope: bazel-test - name: Observe resources after preparation and input installation continue-on-error: true From 460c7799432151a6dc18f28dfa8e805888aca8bc Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 10:21:41 -0400 Subject: [PATCH 17/41] test: bound full voice test listing and remove private fixture plugin --- codex-rs/voice-host/tests/installed_client.rs | 10 +++- .../voice/test_windows_runtime_diagnostic.py | 48 +++++++++++++++++++ .../voice/windows_runtime_diagnostic.py | 10 ++-- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/codex-rs/voice-host/tests/installed_client.rs b/codex-rs/voice-host/tests/installed_client.rs index 42cf42d90497..244c93be0ffa 100644 --- a/codex-rs/voice-host/tests/installed_client.rs +++ b/codex-rs/voice-host/tests/installed_client.rs @@ -141,7 +141,15 @@ async fn installed_client_rejects_mixed_builds_and_missing_helper() -> Result<() .close() .await?; assert!(VoiceHost::connect(&package, "wrong-build").await.is_err()); - // Startup libraries permit handshakes; the missing runtime receipt prevents readiness. + // Unix startup libraries omit the plugin directory. Windows copies plugins + // beside the helper, so remove one required plugin to prevent readiness. + #[cfg(all(windows, target_env = "msvc"))] + fs::remove_file( + helper + .parent() + .context("helper directory")? + .join("gstrtpmanager.dll"), + )?; let host = VoiceHost::connect(&package, &build_commit().await?).await?; assert!(host.initialize_runtime().await.is_err()); // The same executable elsewhere in the package must not become a fallback. diff --git a/third_party/voice/test_windows_runtime_diagnostic.py b/third_party/voice/test_windows_runtime_diagnostic.py index 905d8865283a..f062298f3a28 100644 --- a/third_party/voice/test_windows_runtime_diagnostic.py +++ b/third_party/voice/test_windows_runtime_diagnostic.py @@ -192,6 +192,54 @@ def runner(command, environment, cwd, *args): ) self.assertEqual(output["packaging"]["helper"]["declared_dll_count"], 1) + def test_unit_list_larger_than_build_commit_output_is_bounded_and_required(self): + import io + + real_run = diagnostic.run + for size, valid in [(5000, True), (65537, False)]: + with self.subTest(size=size), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + files, config = self.packaged_fixture(root) + limits = [] + + def runner(command, environment, cwd, *args): + if "--list" in command: + child = unittest.mock.Mock( + stdout=io.BytesIO(b"x" * size), stderr=io.BytesIO(b"") + ) + child.wait.return_value = 0 + with patch.object( + diagnostic.subprocess, "Popen", return_value=child + ): + return real_run(command, environment, cwd, *args) + if "--build-commit" in command: + limits.append(args) + text = "a" * 40 + else: + text = "8664 machine (x64)\nFile Type: EXECUTABLE IMAGE" + return { + "code": 0, + "stdout": text, + "stderr": "", + "truncated": False, + } + + with ( + patch.object( + diagnostic, "resolve", side_effect=lambda key, _: files[key] + ), + patch.object(diagnostic, "run", side_effect=runner), + ): + output = {} + self.assertEqual( + diagnostic.diagnose(config, {"TEST_TMPDIR": str(root)}, output), + valid, + ) + unit = output["probes"][2] + self.assertEqual(len(unit["stdout"]), min(size, 65536)) + self.assertEqual(unit["truncated"], not valid) + self.assertEqual(limits, [(4096,), (4096,)]) + def test_package_check_rejects_missing_modified_dll_and_executable_bytes(self): for change in ["missing", "DLL bytes", "EXE bytes", "receipt"]: with ( diff --git a/third_party/voice/windows_runtime_diagnostic.py b/third_party/voice/windows_runtime_diagnostic.py index f126353c6695..91627c924811 100644 --- a/third_party/voice/windows_runtime_diagnostic.py +++ b/third_party/voice/windows_runtime_diagnostic.py @@ -217,11 +217,11 @@ def diagnose(config, env, output): unrelated.mkdir() copied = fixture(Path(env["TEST_TMPDIR"]) / "installed-fixture", helper, runtime) output["probes"] = [ - run([str(path), arg], env, unrelated) - for path, arg in ( - (helper, "--build-commit"), - (copied, "--build-commit"), - (unit, "--list"), + run([str(path), arg], env, unrelated, limit) + for path, arg, limit in ( + (helper, "--build-commit", 4096), + (copied, "--build-commit", 4096), + (unit, "--list", 65536), ) ] output["environment_identical"] = True From 303b4bf08ea1827821b5122c7a62ab17edec661f Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 10:36:26 -0400 Subject: [PATCH 18/41] ci: supervise the explicit current Git Bash executable --- .../test_windows_ci_resource_observation.py | 74 ++++++++++++++++++- .github/workflows/bazel.yml | 11 ++- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/.github/scripts/test_windows_ci_resource_observation.py b/.github/scripts/test_windows_ci_resource_observation.py index 98783732db71..d4338ba5de49 100644 --- a/.github/scripts/test_windows_ci_resource_observation.py +++ b/.github/scripts/test_windows_ci_resource_observation.py @@ -4,12 +4,15 @@ import importlib.util import io import json +import os from pathlib import Path +import re import subprocess import tempfile import sys import threading import time +import textwrap import unittest from unittest.mock import patch, Mock @@ -80,6 +83,75 @@ def test_start_failure_is_real_and_not_success(self): with self.assertRaises(FileNotFoundError): m.supervise(["missing-build"], None) + @unittest.skipUnless(sys.platform == "win32", "native Windows command selection") + def test_workflow_selects_current_git_bash_and_preserves_native_child_exit(self): + workflow = (Path(__file__).parents[1] / "workflows/bazel.yml").read_text() + step = workflow.split(" - name: Diagnose native MSVC voice runtime\n", 1)[ + 1 + ].split(" - name:", 1)[0] + script = textwrap.dedent(step.split(" run: |\n", 1)[1]) + launch = re.search( + r'python \.github/scripts/observe_windows_ci_resources\.py supervise -- ("[^"\n]+"|[^\s]+)', + script, + ) + self.assertIsNotNone(launch) + # Execute the workflow's real shell selection, without its build payload. + probe = script[: launch.start()] + "printf '%s\\0' " + launch.group(1) + outer = Path(os.environ["ProgramFiles"]) / "Git/bin/bash.exe" + self.assertTrue(outer.is_file(), "Git Bash is required on Windows") + with tempfile.TemporaryDirectory() as directory: + script_path = Path(directory) / "workflow selection.sh" + script_path.write_text(probe, newline="\n") + selected = subprocess.run( + [ + str(outer), + "--noprofile", + "--norc", + "-e", + "-o", + "pipefail", + script_path.as_posix(), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=10, + check=True, + ).stdout + self.assertEqual(selected.count(b"\0"), 1) + executable = Path(selected[:-1].decode()) + self.assertTrue( + executable.is_absolute(), "never resolve a bare bash through PATH" + ) + self.assertTrue(executable.is_file()) + observer = Mock() + with tempfile.TemporaryDirectory() as directory: + marker = Path(directory) / "shell identity.txt" + child_script = Path(directory) / "harmless child.sh" + child_script.write_text( + 'printf "%s\\n" "$BASH_VERSION" "$2" > "$1"\nexit 37\n', + newline="\n", + ) + with patch.object(m, "collect", side_effect=AssertionError("no host CIM")): + code = m.supervise( + [ + str(executable), + "--noprofile", + "--norc", + child_script.as_posix(), + marker.as_posix(), + "argument with spaces", + ], + observer, + ) + self.assertEqual(code, 37) + version, argument = marker.read_text().splitlines() + self.assertTrue(re.fullmatch(r"\d+\.\d+.*", version)) + self.assertEqual(argument, "argument with spaces") + self.assertEqual( + [call.args[0] for call in observer.sample.call_args_list], + ["before-build", "build-exit"], + ) + def test_oversized_collector_rejected(self): with patch.object(m, "bounded_output", side_effect=OverflowError()): self.assertEqual(m.collect(), {"unavailable": 1, "truncated": True}) @@ -197,7 +269,7 @@ def test_workflow_only_diagnostic_job_uses_observer(self): before, job = workflow.read_text().split(" voice-runtime-diagnostic:", 1) self.assertNotIn("observe_windows_ci_resources", before) self.assertIn( - "python .github/scripts/observe_windows_ci_resources.py supervise -- bash ./.github/scripts/run-bazel-ci.sh", + 'python .github/scripts/observe_windows_ci_resources.py supervise -- "$git_bash" ./.github/scripts/run-bazel-ci.sh', job, ) self.assertIn( diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index ed77877d4cf2..34cc9ef423a5 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -729,7 +729,16 @@ jobs: # its native MSVC route. This is not GNU cross-compile evidence. run: | unset BUILDBUDDY_API_KEY - python .github/scripts/observe_windows_ci_resources.py supervise -- bash ./.github/scripts/run-bazel-ci.sh \ + if [[ "$OSTYPE" != msys || "$BASH" != /* || ! -x "$BASH" ]]; then + echo "Expected the current Git Bash executable" >&2 + exit 1 + fi + git_bash="$(cygpath -w "$BASH")" + case "$git_bash" in + [A-Za-z]:\\*|[A-Za-z]:/*) ;; + *) echo "Git Bash did not resolve to an absolute Windows path" >&2; exit 1 ;; + esac + python .github/scripts/observe_windows_ci_resources.py supervise -- "$git_bash" ./.github/scripts/run-bazel-ci.sh \ --print-failed-test-logs --print-failed-action-summary \ --windows-cross-compile -- test --keep_going --nocache_test_results --test_output=all --zip_undeclared_test_outputs=false \ --test_env=VOICE_DIAGNOSTIC_HEAD --test_env=VOICE_DIAGNOSTIC_TREE \ From 987987bec83f10b84479a09341e5ecfb65336012 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 11:30:43 -0400 Subject: [PATCH 19/41] ci: normalize current Bash paths before executable validation --- .../test_windows_ci_resource_observation.py | 130 +++++++++++++----- .github/workflows/bazel.yml | 25 +++- 2 files changed, 115 insertions(+), 40 deletions(-) diff --git a/.github/scripts/test_windows_ci_resource_observation.py b/.github/scripts/test_windows_ci_resource_observation.py index d4338ba5de49..7703981cbcfe 100644 --- a/.github/scripts/test_windows_ci_resource_observation.py +++ b/.github/scripts/test_windows_ci_resource_observation.py @@ -83,8 +83,7 @@ def test_start_failure_is_real_and_not_success(self): with self.assertRaises(FileNotFoundError): m.supervise(["missing-build"], None) - @unittest.skipUnless(sys.platform == "win32", "native Windows command selection") - def test_workflow_selects_current_git_bash_and_preserves_native_child_exit(self): + def workflow_git_bash_probe(self, setup): workflow = (Path(__file__).parents[1] / "workflows/bazel.yml").read_text() step = workflow.split(" - name: Diagnose native MSVC voice runtime\n", 1)[ 1 @@ -96,13 +95,13 @@ def test_workflow_selects_current_git_bash_and_preserves_native_child_exit(self) ) self.assertIsNotNone(launch) # Execute the workflow's real shell selection, without its build payload. - probe = script[: launch.start()] + "printf '%s\\0' " + launch.group(1) + probe = setup + script[: launch.start()] + "printf '%s\\0' " + launch.group(1) outer = Path(os.environ["ProgramFiles"]) / "Git/bin/bash.exe" self.assertTrue(outer.is_file(), "Git Bash is required on Windows") with tempfile.TemporaryDirectory() as directory: script_path = Path(directory) / "workflow selection.sh" script_path.write_text(probe, newline="\n") - selected = subprocess.run( + result = subprocess.run( [ str(outer), "--noprofile", @@ -114,43 +113,100 @@ def test_workflow_selects_current_git_bash_and_preserves_native_child_exit(self) ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=dict(os.environ, ISSUE67_NON_SHELL=sys.executable), timeout=10, - check=True, - ).stdout - self.assertEqual(selected.count(b"\0"), 1) - executable = Path(selected[:-1].decode()) - self.assertTrue( - executable.is_absolute(), "never resolve a bare bash through PATH" - ) - self.assertTrue(executable.is_file()) - observer = Mock() - with tempfile.TemporaryDirectory() as directory: - marker = Path(directory) / "shell identity.txt" - child_script = Path(directory) / "harmless child.sh" - child_script.write_text( - 'printf "%s\\n" "$BASH_VERSION" "$2" > "$1"\nexit 37\n', - newline="\n", + check=False, ) - with patch.object(m, "collect", side_effect=AssertionError("no host CIM")): - code = m.supervise( - [ - str(executable), - "--noprofile", - "--norc", - child_script.as_posix(), - marker.as_posix(), - "argument with spaces", - ], - observer, - ) - self.assertEqual(code, 37) - version, argument = marker.read_text().splitlines() - self.assertTrue(re.fullmatch(r"\d+\.\d+.*", version)) - self.assertEqual(argument, "argument with spaces") + line, separator, selected = result.stdout.partition(b"\n") + self.assertEqual(separator, b"\n") + prefix = b"git-bash-selection " + self.assertTrue(line.startswith(prefix)) + diagnostics = json.loads(line[len(prefix) :]) self.assertEqual( - [call.args[0] for call in observer.sample.call_args_list], - ["before-build", "build-exit"], + set(diagnostics), + { + "original_ostype_msys", + "original_bash_posix_absolute", + "original_bash_executable", + "path_form", + }, + ) + for key in diagnostics.keys() - {"path_form"}: + self.assertIs(type(diagnostics[key]), bool) + self.assertIn( + diagnostics["path_form"], + ("relative", "posix_absolute", "windows_absolute"), ) + return result, diagnostics, selected + + @unittest.skipUnless(sys.platform == "win32", "native Windows command selection") + def test_workflow_selects_current_git_bash_and_preserves_native_child_exit(self): + executables = [] + for setup, form in ( + ('BASH="$(cygpath -au "$BASH")"\n', "posix_absolute"), + ( + 'BASH="$(cygpath -aw "$BASH")"\nOSTYPE=fixture\n', + "windows_absolute", + ), + ): + with self.subTest(form=form): + result, diagnostics, selected = self.workflow_git_bash_probe(setup) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(diagnostics["path_form"], form) + if form == "windows_absolute": + self.assertFalse(diagnostics["original_ostype_msys"]) + self.assertFalse(diagnostics["original_bash_posix_absolute"]) + self.assertEqual(selected.count(b"\0"), 1) + executable = Path(selected[:-1].decode()) + self.assertTrue( + executable.is_absolute(), "never resolve a bare bash through PATH" + ) + self.assertTrue(executable.is_file()) + executables.append(executable) + observer = Mock() + with tempfile.TemporaryDirectory() as directory: + marker = Path(directory) / "shell identity.txt" + child_script = Path(directory) / "harmless child.sh" + child_script.write_text( + 'printf "%s\\n" "$BASH_VERSION" "$2" > "$1"\nexit 37\n', + newline="\n", + ) + with patch.object( + m, "collect", side_effect=AssertionError("no host CIM") + ): + code = m.supervise( + [ + str(executable), + "--noprofile", + "--norc", + child_script.as_posix(), + marker.as_posix(), + "argument with spaces", + ], + observer, + ) + self.assertEqual(code, 37) + version, argument = marker.read_text().splitlines() + self.assertTrue(re.fullmatch(r"\d+\.\d+.*", version)) + self.assertEqual(argument, "argument with spaces") + self.assertEqual( + [call.args[0] for call in observer.sample.call_args_list], + ["before-build", "build-exit"], + ) + self.assertTrue(executables[0].samefile(executables[1])) + + @unittest.skipUnless(sys.platform == "win32", "native Windows command selection") + def test_workflow_rejects_relative_missing_and_non_shell_executables(self): + for setup in ( + "BASH=bash\n", + 'BASH="$(cygpath -aw "$BASH").does-not-exist"\n', + 'BASH="$(cygpath -aw "$ISSUE67_NON_SHELL")"\n', + ): + with self.subTest(setup=setup): + result, diagnostics, selected = self.workflow_git_bash_probe(setup) + self.assertEqual(result.returncode, 1) + self.assertEqual(selected, b"") + self.assertNotIn(sys.executable.encode(), result.stdout + result.stderr) def test_oversized_collector_rejected(self): with patch.object(m, "bounded_output", side_effect=OverflowError()): diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 34cc9ef423a5..128286e223d0 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -729,15 +729,34 @@ jobs: # its native MSVC route. This is not GNU cross-compile evidence. run: | unset BUILDBUDDY_API_KEY - if [[ "$OSTYPE" != msys || "$BASH" != /* || ! -x "$BASH" ]]; then - echo "Expected the current Git Bash executable" >&2 + original_ostype_msys=false + original_bash_posix_absolute=false + original_bash_executable=false + if [[ "${OSTYPE:-}" == msys ]]; then original_ostype_msys=true; fi + if [[ "${BASH:-}" == /* ]]; then original_bash_posix_absolute=true; fi + if [[ -x "${BASH:-}" ]]; then original_bash_executable=true; fi + bash_path_form=relative + case "${BASH:-}" in + /*) bash_path_form=posix_absolute ;; + [A-Za-z]:\\*|[A-Za-z]:/*) bash_path_form=windows_absolute ;; + esac + printf 'git-bash-selection {"original_ostype_msys":%s,"original_bash_posix_absolute":%s,"original_bash_executable":%s,"path_form":"%s"}\n' \ + "$original_ostype_msys" "$original_bash_posix_absolute" "$original_bash_executable" "$bash_path_form" + if [[ -z "${BASH_VERSION:-}" || "$bash_path_form" == relative ]]; then + echo "Expected an absolute current Bash executable" >&2 exit 1 fi - git_bash="$(cygpath -w "$BASH")" + git_bash="$(cygpath -aw "$BASH" 2>/dev/null)" case "$git_bash" in [A-Za-z]:\\*|[A-Za-z]:/*) ;; *) echo "Git Bash did not resolve to an absolute Windows path" >&2; exit 1 ;; esac + git_bash_posix="$(cygpath -au "$git_bash" 2>/dev/null)" + git_bash_name="${git_bash_posix##*/}" + if [[ "$git_bash_posix" != /* || ! -f "$git_bash_posix" || ! -x "$git_bash_posix" || ( "${git_bash_name,,}" != bash.exe && "${git_bash_name,,}" != bash ) ]]; then + echo "Current Bash executable validation failed" >&2 + exit 1 + fi python .github/scripts/observe_windows_ci_resources.py supervise -- "$git_bash" ./.github/scripts/run-bazel-ci.sh \ --print-failed-test-logs --print-failed-action-summary \ --windows-cross-compile -- test --keep_going --nocache_test_results --test_output=all --zip_undeclared_test_outputs=false \ From a339488b309459ca7753ba3b153edeb7555eb5b9 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 09:33:44 -0400 Subject: [PATCH 20/41] ci: avoid duplicate disk cache on keyless hosted Windows --- .github/scripts/hosted_storage.py | 13 ++-- .github/scripts/run-bazel-ci.sh | 9 ++- .github/scripts/test_hosted_storage.py | 90 ++++++++++++++++++++++++-- 3 files changed, 103 insertions(+), 9 deletions(-) diff --git a/.github/scripts/hosted_storage.py b/.github/scripts/hosted_storage.py index b271e10c98aa..2426b4385fa5 100644 --- a/.github/scripts/hosted_storage.py +++ b/.github/scripts/hosted_storage.py @@ -33,17 +33,22 @@ REMOTE_OPTIONS = {"--remote_executor", "--remote_cache", "--strategy"} -def scoped(env): +def policy_scoped(env): return ( env.get("GITHUB_ACTIONS") == "true" and env.get("GITHUB_REPOSITORY") == FORK and env.get("RUNNER_ENVIRONMENT") == "github-hosted" - and env.get("RUNNER_OS") in ("Linux", "macOS") + and env.get("RUNNER_OS") in ("Linux", "macOS", "Windows") and not env.get("BUILDBUDDY_API_KEY") and env.get("HOSTED_STORAGE_REMOTE_CREDENTIALS") != "true" ) +def scoped(env): + """Unix observation/query scope; Windows uses only the argument policy.""" + return policy_scoped(env) and env.get("RUNNER_OS") in ("Linux", "macOS") + + def options(args, names=INFO_OPTIONS): """Read option/value pairs only before payload; retain repeats and empties.""" index = 1 # the Bazel command @@ -75,7 +80,7 @@ def defaults(args, env): if name == "--strategy" and equal: strategies[mnemonic] = choices.split(",") if ( - scoped(env) + policy_scoped(env) and args and args[0] in ("build", "test", "coverage", "run", "info") and not any(name == "--disk_cache" for name, _ in explicit) @@ -96,7 +101,7 @@ def local_arguments(args, env): Other routes retain their original bytes. Only configs already classified by the backend as credential-dependent remote CI are canonicalized and removed. """ - if not scoped(env): + if not policy_scoped(env): return list(args) normalized = [] index = 0 diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index a0c224c5b694..f90dc263a2db 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -8,9 +8,14 @@ remote_download_toplevel=0 windows_msvc_host_platform=0 windows_cross_compile=0 hosted_storage=0 +hosted_storage_policy=0 if [[ "${GITHUB_ACTIONS:-}" == true && "${GITHUB_REPOSITORY:-}" == CompleteDotTech/codex-subagents-multi-provider && "${RUNNER_ENVIRONMENT:-}" == github-hosted && ( "${RUNNER_OS:-}" == Linux || "${RUNNER_OS:-}" == macOS ) && -z "${BUILDBUDDY_API_KEY:-}" ]]; then hosted_storage=1 fi +# Windows shares only the cache argument policy, not Unix observation/query APIs. +if [[ $hosted_storage -eq 1 || ( "${GITHUB_ACTIONS:-}" == true && "${GITHUB_REPOSITORY:-}" == CompleteDotTech/codex-subagents-multi-provider && "${RUNNER_ENVIRONMENT:-}" == github-hosted && "${RUNNER_OS:-}" == Windows && -z "${BUILDBUDDY_API_KEY:-}" && "${HOSTED_STORAGE_REMOTE_CREDENTIALS:-}" != true ) ]]; then + hosted_storage_policy=1 +fi storage_script="$(dirname "${BASH_SOURCE[0]}")/hosted_storage.py" while [[ $# -gt 0 ]]; do @@ -563,7 +568,7 @@ fi if (( ${#post_config_bazel_args[@]} > 0 )); then bazel_run_args+=("${post_config_bazel_args[@]}") fi -if [[ $hosted_storage -eq 1 ]]; then +if [[ $hosted_storage_policy -eq 1 ]]; then # Use the backend's keyless config classification before deciding defaults; # BUILD and INFO must see the same effective local arguments. effective_bazel_args=() @@ -575,6 +580,8 @@ if [[ $hosted_storage -eq 1 ]]; then exit 1 fi bazel_run_args=("${effective_bazel_args[@]}") +fi +if [[ $hosted_storage -eq 1 ]]; then python3 "$storage_script" observe before-build none "${bazel_run_args[@]}" || echo "Hosted storage observation unavailable" >&2 fi set +e diff --git a/.github/scripts/test_hosted_storage.py b/.github/scripts/test_hosted_storage.py index ae86e8ce5cc7..1b105ee7447a 100644 --- a/.github/scripts/test_hosted_storage.py +++ b/.github/scripts/test_hosted_storage.py @@ -27,7 +27,7 @@ class PolicyTests(unittest.TestCase): def test_default_is_scoped_and_keeps_remote_configs_out(self): - for os_name in ("Linux", "macOS"): + for os_name in ("Linux", "macOS", "Windows"): self.assertEqual( storage.defaults(["test"], dict(ENV, RUNNER_OS=os_name)), ["--disk_cache="], @@ -36,7 +36,7 @@ def test_default_is_scoped_and_keeps_remote_configs_out(self): {"GITHUB_ACTIONS": "false"}, {"GITHUB_REPOSITORY": "openai/codex"}, {"RUNNER_ENVIRONMENT": "self-hosted"}, - {"RUNNER_OS": "Windows"}, + {"RUNNER_OS": "FreeBSD"}, {"BUILDBUDDY_API_KEY": "synthetic"}, ): self.assertEqual(storage.defaults(["test"], dict(ENV, **change)), []) @@ -80,6 +80,34 @@ def test_local_only_strategies_and_last_value_for_same_mnemonic(self): ): self.assertEqual(storage.defaults(["test", *flags], ENV), []) + def test_windows_policy_preserves_explicit_routes_and_keeps_observation_disabled( + self, + ): + env = dict(ENV, RUNNER_OS="Windows") + self.assertFalse(storage.scoped(env)) + for flags in ( + ["--disk_cache="], + ["--disk_cache", "owned cache"], + ["--disk_cache=first", "--disk_cache", "last"], + ["--config=buildbuddy-generic-rbe"], + ["--config", "remote"], + ["--remote_executor=grpcs://synthetic"], + ["--strategy=Rustc=remote"], + ): + args = ["test", *flags, "--", "//target", "--disk_cache=payload"] + self.assertEqual(storage.effective_arguments(args, env), args) + for change in ( + {"BUILDBUDDY_API_KEY": "synthetic"}, + {"HOSTED_STORAGE_REMOTE_CREDENTIALS": "true"}, + {"GITHUB_REPOSITORY": "openai/codex"}, + {"GITHUB_ACTIONS": "false"}, + {"RUNNER_ENVIRONMENT": "self-hosted"}, + ): + self.assertEqual( + storage.effective_arguments(["test"], dict(env, **change)), ["test"] + ) + self.assertEqual(storage.actual_paths(["test"], env), ("out-of-scope", None)) + class BackendInteractionTests(unittest.TestCase): def test_final_endpoint_values_control_local_default_without_rewriting_flags(self): @@ -197,7 +225,7 @@ def test_new_normalization_does_not_change_other_routes(self): for change in ( {"BUILDBUDDY_API_KEY": "synthetic-key"}, {"GITHUB_REPOSITORY": "openai/codex"}, - {"RUNNER_OS": "Windows"}, + {"RUNNER_OS": "FreeBSD"}, {"GITHUB_ACTIONS": "false"}, ): env = dict(ENV, **change) @@ -721,7 +749,7 @@ def test_numeric_summary_survives_receipt_disk_write_failure(self): class ShellTests(unittest.TestCase): - def invoke(self, flags=(), extra=()): + def invoke(self, flags=(), extra=(), environment=None): bash = ( str( Path(os.environ.get("ProgramFiles", "C:/Program Files")) @@ -757,6 +785,7 @@ def invoke(self, flags=(), extra=()): " if 'info' in sys.argv: print('/nonexistent-testlogs'); sys.exit(0)\n" " sys.exit(37)\n", encoding="utf-8", + newline="\n", ) (root / "run_bazel_with_buildbuddy.py").chmod(0o755) record = root / "record" @@ -766,6 +795,9 @@ def invoke(self, flags=(), extra=()): if key in ("PATH", "SYSTEMROOT", "SystemRoot", "TEMP", "TMP") } env.update(ENV, RECORD=str(record), MSYS2_ARG_CONV_EXCL="*") + env.update(environment or {}) + if env["RUNNER_OS"] == "Windows": + env["CODEX_BAZEL_WINDOWS_PATH"] = "synthetic-no-execution" result = subprocess.run( [ bash, @@ -784,10 +816,60 @@ def invoke(self, flags=(), extra=()): text=True, timeout=20, ) + self.assertTrue(record.exists(), result.stderr) return result, [ json.loads(line) for line in record.read_text().splitlines() ] + def test_windows_policy_routes_without_unix_observers_or_queries(self): + for flags, expected in ( + ([], ["--disk_cache="]), + (["--disk_cache=owned"], ["--disk_cache=owned"]), + (["--disk_cache", ""], ["--disk_cache", ""]), + ( + ["--disk_cache=first", "--disk_cache", "last"], + ["--disk_cache=first", "--disk_cache", "last"], + ), + ( + ["--remote_cache=https://synthetic"], + ["--remote_cache=https://synthetic"], + ), + ): + with self.subTest(flags=flags): + result, calls = self.invoke( + flags, ["--disk_cache=payload"], {"RUNNER_OS": "Windows"} + ) + self.assertEqual(result.returncode, 37, result.stderr) + self.assertEqual(len(calls), 2, calls) + build, info = calls + self.assertEqual( + build[build.index("test") + 1 : build.index("--")], + expected + + [ + "--repo_env=BAZEL_MSVC_RUNTIME_VISUAL_STUDIO_EULA=1", + "--action_env=PATH=synthetic-no-execution", + "--host_action_env=PATH=synthetic-no-execution", + "--test_env=PATH=synthetic-no-execution", + ], + ) + self.assertEqual( + build[build.index("--") + 1 :], ["//target", "--disk_cache=payload"] + ) + self.assertIn("info", info) + self.assertNotIn("Hosted Bazel paths unavailable", result.stderr) + + def test_windows_credentials_leave_wrapper_policy_unchanged(self): + for credentials in ( + {"BUILDBUDDY_API_KEY": "synthetic"}, + {"HOSTED_STORAGE_REMOTE_CREDENTIALS": "true"}, + ): + result, calls = self.invoke( + environment=dict(credentials, RUNNER_OS="Windows") + ) + self.assertEqual(result.returncode, 37, result.stderr) + self.assertEqual(len(calls), 2, calls) + self.assertNotIn("--disk_cache=", calls[0]) + def test_failure_snapshot_precedes_info_and_original_exit_survives(self): result, calls = self.invoke( ["--compilation_mode", "opt", "--platforms", "//:custom"] From 7997a76ca7a0928715088018faa1c3d2a822233c Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 22:28:04 -0400 Subject: [PATCH 21/41] fix: annotate inherited provider arguments for strict lint gates Refs #50. Preserve provider authority values and assertions while resolving the inherited exact-base argument-comment and config Clippy diagnostics. (cherry picked from commit 5f3e9603c92b75f1d2c8edba0ebdb35583b35817) --- codex-rs/config/src/child_provider.rs | 10 ++- codex-rs/config/src/child_provider_tests.rs | 57 +++++++------- codex-rs/core/src/agent/child_config.rs | 2 +- codex-rs/core/src/agent/child_config_tests.rs | 58 +++++++++----- .../src/models_endpoint_gateway_tests.rs | 78 +++++++++++++------ .../src/cache_identity_tests.rs | 31 +++++--- 6 files changed, 152 insertions(+), 84 deletions(-) diff --git a/codex-rs/config/src/child_provider.rs b/codex-rs/config/src/child_provider.rs index bf69001ae5fe..b4400bbd140e 100644 --- a/codex-rs/config/src/child_provider.rs +++ b/codex-rs/config/src/child_provider.rs @@ -85,7 +85,7 @@ impl VerifiedChildProviderOrigin { binding: &ChildProviderOriginBinding, ) -> Result { if record.origin_id.is_nil() - || record.root_thread_id == ThreadId::from_u128(0) + || record.root_thread_id == ThreadId::from_u128(/*value*/ 0) || !valid_id(&record.provider_id) || record.origin_id != binding.origin_id || record.root_thread_id != binding.root_thread_id @@ -156,7 +156,11 @@ impl ChildProviderPolicy { file_system_policy: &FileSystemSandboxPolicy, ) -> std::io::Result { let mut project_roots = vec![cwd.to_path_buf()]; - project_roots.extend(workspace_roots.iter().map(|root| root.to_path_buf())); + project_roots.extend( + workspace_roots + .iter() + .map(codex_utils_absolute_path::AbsolutePathBuf::to_path_buf), + ); for layer in stack.layers_low_to_high() { if let ConfigLayerSource::Project { dot_codex_folder } = &layer.name && let Some(root) = dot_codex_folder.as_path().parent() @@ -320,7 +324,7 @@ impl ChildProviderPolicy { cwd: cwd.clone(), protected_catalog_roots: protected_catalog_roots .iter() - .map(|root| root.to_path_buf()) + .map(codex_utils_absolute_path::AbsolutePathBuf::to_path_buf) .collect(), }) } diff --git a/codex-rs/config/src/child_provider_tests.rs b/codex-rs/config/src/child_provider_tests.rs index a0093d60f74d..cf03294d83c2 100644 --- a/codex-rs/config/src/child_provider_tests.rs +++ b/codex-rs/config/src/child_provider_tests.rs @@ -121,7 +121,7 @@ fn request<'a>(parent: &'a ModelProviderInfo) -> ChildProviderRequest<'a> { fn origin(provider_id: &str) -> VerifiedChildProviderOrigin { let record = ChildProviderOriginRecord { origin_id: Uuid::from_u128(1), - root_thread_id: ThreadId::from_u128(2), + root_thread_id: ThreadId::from_u128(/*value*/ 2), provider_id: provider_id.to_owned(), }; let binding = ChildProviderOriginBinding { @@ -136,7 +136,7 @@ fn origin(provider_id: &str) -> VerifiedChildProviderOrigin { fn omitted_fields_keep_legacy_parent_definition_and_model() { let mut fixture = Fixture::new(); fixture.user.clear(); - let policy = fixture.policy("", "", None); + let policy = fixture.policy("", "", /*required*/ None); let parent = ModelProviderInfo { name: "existing root".into(), ..Default::default() @@ -154,7 +154,7 @@ fn explicit_provider_beats_role_and_host_default_while_role_model_wins() { "allowed_model_providers =", "default_subagent_model_provider = \"ollama\"\nallowed_model_providers =", ); - let policy = fixture.policy("", "", None); + let policy = fixture.policy("", "", /*required*/ None); let parent = ModelProviderInfo::default(); let mut input = request(&parent); input.explicit_provider = Some("zai"); @@ -173,7 +173,7 @@ fn explicit_provider_beats_role_and_host_default_while_role_model_wins() { #[test] fn target_defaults_beat_incompatible_parent_and_global_model() { let fixture = Fixture::new(); - let policy = fixture.policy("", "", None); + let policy = fixture.policy("", "", /*required*/ None); let parent = ModelProviderInfo::default(); for id in ["zai", "ollama"] { let mut input = request(&parent); @@ -192,7 +192,7 @@ fn role_then_host_default_then_parent_precedence() { "allowed_model_providers =", "default_subagent_model_provider = \"ollama\"\nallowed_model_providers =", ); - let policy = fixture.policy("", "", None); + let policy = fixture.policy("", "", /*required*/ None); let parent = ModelProviderInfo::default(); assert_eq!( policy.resolve(request(&parent)).unwrap().provider_id, @@ -206,7 +206,7 @@ fn role_then_host_default_then_parent_precedence() { #[test] fn explicit_return_to_root_requires_no_nonroot_allowlist_entry() { let fixture = Fixture::new(); - let mut policy = fixture.policy("", "", None); + let mut policy = fixture.policy("", "", /*required*/ None); policy.allowed.clear(); let parent = ModelProviderInfo::default(); let mut input = request(&parent); @@ -219,7 +219,7 @@ fn explicit_return_to_root_requires_no_nonroot_allowlist_entry() { #[test] fn nested_provider_inherits_immediate_parent_but_revocation_still_applies() { let fixture = Fixture::new(); - let policy = fixture.policy("", "", None); + let policy = fixture.policy("", "", /*required*/ None); let parent = policy.providers["zai"].clone(); let mut input = request(&parent); input.parent_provider_id = "zai"; @@ -238,7 +238,10 @@ fn nested_provider_inherits_immediate_parent_but_revocation_still_applies() { ); assert!(revoked.authorize("openai", Some(&origin("openai"))).is_ok()); assert_eq!( - revoked.authorize("openai", None).unwrap_err().code, + revoked + .authorize("openai", /*origin*/ None) + .unwrap_err() + .code, "child_provider_not_authorized" ); revoked.allowed.insert("zai".to_owned()); @@ -258,7 +261,7 @@ fn project_and_session_cannot_authorize_or_change_defaults() { fixture.user.clear(); let hostile = "[agents]\nallowed_model_providers=['zai']\ndefault_subagent_model_provider='zai'"; - let policy = fixture.policy(hostile, hostile, None); + let policy = fixture.policy(hostile, hostile, /*required*/ None); assert_eq!(policy.default_provider(), None); assert_eq!( policy @@ -283,7 +286,7 @@ fn unrelated_trusted_settings_can_be_corrected_by_project_without_affecting_poli let _guard = codex_utils_absolute_path::AbsolutePathBufGuard::new(fixture.home.path()); assert!(effective.try_into::().is_ok()); } - let policy = fixture.policy(project, "", None); + let policy = fixture.policy(project, "", /*required*/ None); assert!(policy.authorize("zai", Some(&origin("openai"))).is_ok()); } @@ -293,7 +296,7 @@ fn same_id_project_and_session_provider_injection_is_denied_before_helpers() { let hostile = "[model_providers.zai]\nname='attacker'\nbase_url='https://attacker.invalid'\n[model_providers.zai.auth]\ncommand='MUST_NOT_EXECUTE'"; let parent = ModelProviderInfo::default(); for (project, session) in [(hostile, ""), ("", hostile)] { - let policy = fixture.policy(project, session, None); + let policy = fixture.policy(project, session, /*required*/ None); assert_eq!( policy.providers["zai"].base_url.as_deref(), Some("https://trusted.example/v1") @@ -325,7 +328,7 @@ fn managed_provider_constraint_rejects_even_authorized_override() { fn missing_model_catalog_and_unknown_provider_fail_without_fallback() { let mut fixture = Fixture::new(); fixture.user = fixture.user.replace("model = \"shared\"", ""); - let policy = fixture.policy("", "", None); + let policy = fixture.policy("", "", /*required*/ None); let parent = ModelProviderInfo::default(); let mut input = request(&parent); input.explicit_provider = Some("zai"); @@ -336,7 +339,7 @@ fn missing_model_catalog_and_unknown_provider_fail_without_fallback() { fixture.user = fixture .user .replace("model_catalog_json = \"catalog.json\"", ""); - let mut policy = fixture.policy("", "", None); + let mut policy = fixture.policy("", "", /*required*/ None); let mut input = request(&parent); input.explicit_provider = Some("zai"); input.explicit_model = Some("shared"); @@ -356,7 +359,7 @@ fn missing_model_catalog_and_unknown_provider_fail_without_fallback() { #[test] fn cross_provider_full_history_rejects_before_catalog_lookup() { let fixture = Fixture::new(); - let policy = fixture.policy("", "", None); + let policy = fixture.policy("", "", /*required*/ None); std::fs::remove_file(fixture.home.path().join("catalog.json")).unwrap(); let parent = ModelProviderInfo::default(); let mut input = request(&parent); @@ -371,7 +374,7 @@ fn cross_provider_full_history_rejects_before_catalog_lookup() { #[test] fn same_id_changed_definition_rejects_full_history_before_catalog_lookup() { let fixture = Fixture::new(); - let policy = fixture.policy("", "", None); + let policy = fixture.policy("", "", /*required*/ None); std::fs::remove_file(fixture.home.path().join("catalog.json")).unwrap(); for change_endpoint in [true, false] { let mut parent = policy.providers["zai"].clone(); @@ -399,7 +402,7 @@ fn catalog_under_project_is_not_trusted_by_a_user_config_reference() { "catalog.json", &catalog.to_string_lossy().replace('\\', "/"), ); - let policy = fixture.policy("", "", None); + let policy = fixture.policy("", "", /*required*/ None); let parent = ModelProviderInfo::default(); let mut input = request(&parent); input.explicit_provider = Some("zai"); @@ -416,7 +419,7 @@ fn ancestor_dotfiles_repo_does_not_taint_host_catalog_but_nested_repo_does() { let mut fixture = Fixture::new(); fixture.home = tempfile::tempdir_in(dotfiles.path()).unwrap(); std::fs::write(fixture.home.path().join("catalog.json"), "{}").unwrap(); - let policy = fixture.policy_with_protected_roots("", "", None, &[]); + let policy = fixture.policy_with_protected_roots("", "", /*required*/ None, &[]); let parent = ModelProviderInfo::default(); let mut input = request(&parent); input.explicit_provider = Some("zai"); @@ -446,7 +449,7 @@ fn host_catalog_in_active_workspace_requires_actual_read_only_protection() { } else { vec![] }; - let policy = fixture.policy_with_protected_roots("", "", None, &roots); + let policy = fixture.policy_with_protected_roots("", "", /*required*/ None, &roots); let parent = ModelProviderInfo::default(); let mut input = request(&parent); input.explicit_provider = Some("zai"); @@ -477,7 +480,7 @@ fn writable_user_source_cannot_authorize_url_catalogs_endpoints_or_helpers() { "https://reloaded-attacker.invalid/v1", ] { fixture.user = fixture.user.replace("https://trusted.example/v1", endpoint); - let policy = fixture.policy_with_protected_roots("", "", None, &[]); + let policy = fixture.policy_with_protected_roots("", "", /*required*/ None, &[]); assert!(policy.allowed.is_empty()); assert!(!policy.providers.contains_key("zai")); assert!(policy.defaults.is_empty()); @@ -500,7 +503,7 @@ fn writable_user_source_cannot_authorize_url_catalogs_endpoints_or_helpers() { } // The same source contributes authority only when the actual host policy protects it. let protected = AbsolutePathBuf::from_absolute_path(fixture.home.path()).unwrap(); - let policy = fixture.policy_with_protected_roots("", "", None, &[protected]); + let policy = fixture.policy_with_protected_roots("", "", /*required*/ None, &[protected]); assert!(policy.allowed.contains("zai")); assert_eq!( policy.providers["zai"].base_url.as_deref(), @@ -554,7 +557,7 @@ fn writable_user_without_child_authority_preserves_legacy_configuration() { let mut fixture = Fixture::new(); fixture.home = tempfile::tempdir_in(fixture.project.path()).unwrap(); fixture.user = "model='legacy-model'\n[agents]\nmax_depth=3".to_owned(); - let policy = fixture.policy_with_protected_roots("", "", None, &[]); + let policy = fixture.policy_with_protected_roots("", "", /*required*/ None, &[]); let parent = ModelProviderInfo::default(); let result = policy.resolve(request(&parent)).unwrap(); assert_eq!(result.provider, parent); @@ -568,7 +571,7 @@ fn user_source_symlink_into_writable_scope_cannot_supply_authority() { let target = fixture.project.path().join("controlled.toml"); std::fs::write(&target, &fixture.user).unwrap(); std::os::unix::fs::symlink(&target, fixture.home.path().join("config.toml")).unwrap(); - let policy = fixture.policy_with_protected_roots("", "", None, &[]); + let policy = fixture.policy_with_protected_roots("", "", /*required*/ None, &[]); assert!(policy.allowed.is_empty()); assert!(!policy.providers.contains_key("zai")); } @@ -641,7 +644,7 @@ fn host_owned_children_ignore_user_default_and_reject_explicit_selection() { "allowed_model_providers =", "default_subagent_model_provider = \"zai\"\nallowed_model_providers =", ); - let policy = fixture.policy("", "", None); + let policy = fixture.policy("", "", /*required*/ None); let parent = ModelProviderInfo::default(); let mut input = request(&parent); input.host_owned = true; @@ -666,7 +669,7 @@ fn host_owned_children_ignore_user_default_and_reject_explicit_selection() { #[test] fn same_slug_catalogs_keep_provider_metadata_separate_and_reject_fallback() { let fixture = Fixture::new(); - let policy = fixture.policy("", "", None); + let policy = fixture.policy("", "", /*required*/ None); let parent = ModelProviderInfo::default(); let catalog: ModelsResponse = serde_json::from_str(include_str!("../../models-manager/models.json")).unwrap(); @@ -810,7 +813,7 @@ fn managed_allowlists_intersect_and_cannot_be_broadened_by_user_config() { fn verified_origin_requires_matching_uuid_root_and_provider_binding() { let record = ChildProviderOriginRecord { origin_id: Uuid::from_u128(1), - root_thread_id: ThreadId::from_u128(2), + root_thread_id: ThreadId::from_u128(/*value*/ 2), provider_id: "openai".into(), }; let binding = ChildProviderOriginBinding { @@ -824,7 +827,7 @@ fn verified_origin_requires_matching_uuid_root_and_provider_binding() { assert_eq!(verified.root_thread_id(), record.root_thread_id); let mut mismatches = vec![binding.clone(); 3]; mismatches[0].origin_id = Uuid::from_u128(3); - mismatches[1].root_thread_id = ThreadId::from_u128(3); + mismatches[1].root_thread_id = ThreadId::from_u128(/*value*/ 3); mismatches[2].provider_id = "zai".into(); for mismatch in mismatches { assert_eq!( @@ -834,7 +837,7 @@ fn verified_origin_requires_matching_uuid_root_and_provider_binding() { "child_provider_policy_conflict" ); } - let mut new_root_origin = record.clone(); + let mut new_root_origin = record; new_root_origin.origin_id = Uuid::from_u128(4); new_root_origin.provider_id = "zai".into(); // A later valid root origin cannot silently rebind an existing child. diff --git a/codex-rs/core/src/agent/child_config.rs b/codex-rs/core/src/agent/child_config.rs index 7f29ca3428be..d11100bae9dd 100644 --- a/codex-rs/core/src/agent/child_config.rs +++ b/codex-rs/core/src/agent/child_config.rs @@ -549,7 +549,7 @@ pub(crate) async fn apply_spawn_agent_service_tier( session, &session.services.models_manager, config, - false, + /*strict_catalog*/ false, ) .await } diff --git a/codex-rs/core/src/agent/child_config_tests.rs b/codex-rs/core/src/agent/child_config_tests.rs index 56f8bbbdf76e..4cff1b0fb24f 100644 --- a/codex-rs/core/src/agent/child_config_tests.rs +++ b/codex-rs/core/src/agent/child_config_tests.rs @@ -34,7 +34,7 @@ impl Fixture { .thread } async fn new() -> Self { - Self::with_fast_mode(None).await + Self::with_fast_mode(/*fast_mode*/ None).await } async fn with_fast_mode(fast_mode: Option) -> Self { @@ -209,8 +209,8 @@ async fn selected_same_slug_catalog_reaches_startup_without_changing_parent_or_s let zai = fixture.resolved("zai").await; let ollama = fixture.resolved("ollama").await; let (zai, ollama) = tokio::join!( - fixture.prepared(&zai, None, None), - fixture.prepared(&ollama, None, None) + fixture.prepared(&zai, /*role*/ None, /*effort*/ None), + fixture.prepared(&ollama, /*role*/ None, /*effort*/ None) ); let zai = zai.unwrap().config; let ollama = ollama.unwrap().config; @@ -265,7 +265,7 @@ async fn prepared_scope_cannot_survive_provider_catalog_model_or_auth_changes() let fixture = Fixture::new().await; let resolved = fixture.resolved("zai").await; let prepared = fixture - .prepared(&resolved, None, None) + .prepared(&resolved, /*role*/ None, /*effort*/ None) .await .unwrap() .config; @@ -299,7 +299,7 @@ async fn unknown_model_and_unsupported_explicit_effort_fail_without_parent_fallb let fixture = Fixture::new().await; let mut resolved = fixture.resolved("zai").await; let result = fixture - .prepared(&resolved, None, Some(ReasoningEffort::High)) + .prepared(&resolved, /*role*/ None, Some(ReasoningEffort::High)) .await; assert!( result @@ -308,7 +308,9 @@ async fn unknown_model_and_unsupported_explicit_effort_fail_without_parent_fallb .contains("child_provider_capability_unsupported") ); resolved.model = "gpt-5.5".into(); - let result = fixture.prepared(&resolved, None, None).await; + let result = fixture + .prepared(&resolved, /*role*/ None, /*effort*/ None) + .await; assert!( result .err() @@ -321,7 +323,9 @@ async fn unknown_model_and_unsupported_explicit_effort_fail_without_parent_fallb async fn failed_target_catalog_discovery_cannot_validate_a_bundled_openai_slug() { let fixture = Fixture::new().await; let resolved = fixture.resolved("offline").await; - let result = fixture.prepared(&resolved, None, None).await; + let result = fixture + .prepared(&resolved, /*role*/ None, /*effort*/ None) + .await; assert!( result .err() @@ -362,7 +366,9 @@ async fn prepared_provider_selection_cannot_be_reused_for_a_full_history_switch( async fn explicit_role_tier_is_validated_even_when_equal_to_inherited_value() { let fixture = Fixture::new().await; let resolved = fixture.resolved("zai").await; - let result = fixture.prepared(&resolved, Some("tier"), None).await; + let result = fixture + .prepared(&resolved, Some("tier"), /*effort*/ None) + .await; assert!( result .err() @@ -370,7 +376,7 @@ async fn explicit_role_tier_is_validated_even_when_equal_to_inherited_value() { .contains("child_provider_capability_unsupported") ); let mut config = fixture - .prepared(&resolved, None, None) + .prepared(&resolved, /*role*/ None, /*effort*/ None) .await .unwrap() .config; @@ -386,7 +392,7 @@ async fn explicit_role_tier_is_validated_even_when_equal_to_inherited_value() { .unwrap(); assert_eq!(explicit, Some("flex".into())); let default = fixture - .prepared(&resolved, Some("default_tier"), None) + .prepared(&resolved, Some("default_tier"), /*effort*/ None) .await .unwrap(); assert_eq!( @@ -416,7 +422,9 @@ async fn advertised_custom_role_tier_requires_the_same_feature_gate_as_runtime() description: "fixture".into(), }); std::fs::write(path, serde_json::to_vec(&catalog).unwrap()).unwrap(); - let result = fixture.prepared(&resolved, Some("tier"), None).await; + let result = fixture + .prepared(&resolved, Some("tier"), /*effort*/ None) + .await; if !enabled { assert!( result @@ -453,7 +461,7 @@ async fn unsupported_inherited_tier_is_omitted_and_default_sentinel_is_preserved let resolved = fixture.resolved("zai").await; assert_eq!( fixture - .prepared(&resolved, None, None) + .prepared(&resolved, /*role*/ None, /*effort*/ None) .await .unwrap() .config @@ -470,7 +478,7 @@ async fn unsupported_inherited_tier_is_omitted_and_default_sentinel_is_preserved ))); assert_eq!( fixture - .prepared(&resolved, None, None) + .prepared(&resolved, /*role*/ None, /*effort*/ None) .await .unwrap() .config @@ -483,7 +491,10 @@ async fn unsupported_inherited_tier_is_omitted_and_default_sentinel_is_preserved async fn native_step_does_not_reintroduce_an_unsupported_root_flex_tier() { let fixture = Fixture::new().await; let resolved = fixture.resolved("zai").await; - let prepared = fixture.prepared(&resolved, None, None).await.unwrap(); + let prepared = fixture + .prepared(&resolved, /*role*/ None, /*effort*/ None) + .await + .unwrap(); let child = fixture.start_native_child(prepared.config).await; child .session @@ -517,7 +528,7 @@ async fn native_explicit_role_tier_survives_root_updates_and_is_not_inherited_as }); std::fs::write(path, serde_json::to_vec(&catalog).unwrap()).unwrap(); let prepared = fixture - .prepared(&resolved, Some("tier"), None) + .prepared(&resolved, Some("tier"), /*effort*/ None) .await .unwrap(); assert!(prepared.config.child_role_service_tier_override); @@ -549,9 +560,15 @@ async fn changed_provider_uses_target_reasoning_default_and_preserves_role_prece let resolved = fixture.resolved("zai").await; let mut config = fixture.config.clone(); config.agent_default_subagent_reasoning_effort = Some(ReasoningEffort::High); - prepare_resolved_child_models(&fixture.parent.session, &mut config, &resolved, None, None) - .await - .unwrap(); + prepare_resolved_child_models( + &fixture.parent.session, + &mut config, + &resolved, + /*requested_model*/ None, + /*requested_effort*/ None, + ) + .await + .unwrap(); assert_eq!(config.model_reasoning_effort, Some(ReasoningEffort::Low)); let prepared = fixture .prepared(&resolved, Some("reasoning"), Some(ReasoningEffort::High)) @@ -567,7 +584,10 @@ async fn changed_provider_uses_target_reasoning_default_and_preserves_role_prece async fn fresh_same_provider_reload_uses_changed_catalog_prompts_and_limits() { let fixture = Fixture::new().await; let resolved = fixture.resolved("zai").await; - let prepared = fixture.prepared(&resolved, None, None).await.unwrap(); + let prepared = fixture + .prepared(&resolved, /*role*/ None, /*effort*/ None) + .await + .unwrap(); let first = fixture .manager .start_thread(StartThreadOptions::new(prepared.config)) diff --git a/codex-rs/model-provider/src/models_endpoint_gateway_tests.rs b/codex-rs/model-provider/src/models_endpoint_gateway_tests.rs index 0b7fb4646841..3c7cd68c4e97 100644 --- a/codex-rs/model-provider/src/models_endpoint_gateway_tests.rs +++ b/codex-rs/model-provider/src/models_endpoint_gateway_tests.rs @@ -82,12 +82,16 @@ async fn gateway_only_catalog_scope_observes_external_accounts_legacy_grants_and ); let endpoint = Arc::new(OpenAiModelsEndpoint::new( info.clone(), - None, + /*auth_manager*/ None, Some(Ok(gateway.clone())), )); - let manager = OpenAiModelsManager::new(home.path().to_path_buf(), endpoint.clone(), None) - .with_provider_catalog(); - mount(&server, "legacy", 1000, Duration::ZERO).await; + let manager = OpenAiModelsManager::new( + home.path().to_path_buf(), + endpoint.clone(), + /*auth_manager*/ None, + ) + .with_provider_catalog(); + mount(&server, "legacy", /*window*/ 1000, Duration::ZERO).await; assert_eq!( manager .raw_model_catalog(RefreshStrategy::OnlineIfUncached, HTTP) @@ -111,7 +115,7 @@ async fn gateway_only_catalog_scope_observes_external_accounts_legacy_grants_and ); assert_ne!(endpoint.identity().unwrap(), legacy_scope); assert!(manager.try_get_remote_models().unwrap().is_empty()); - mount(&server, "account-b", 2000, Duration::ZERO).await; + mount(&server, "account-b", /*window*/ 2000, Duration::ZERO).await; assert_eq!( manager .raw_model_catalog(RefreshStrategy::OnlineIfUncached, HTTP) @@ -134,7 +138,7 @@ async fn gateway_only_catalog_scope_observes_external_accounts_legacy_grants_and .expect(1) .mount(&server) .await; - mount(&server, "rotated-b", 3000, Duration::ZERO).await; + mount(&server, "rotated-b", /*window*/ 3000, Duration::ZERO).await; assert_eq!( manager .raw_model_catalog(RefreshStrategy::OnlineIfUncached, HTTP) @@ -176,13 +180,24 @@ async fn changed_gateway_grant_during_catalog_response_cannot_publish_or_cache_o ); let endpoint = Arc::new(OpenAiModelsEndpoint::new( info.clone(), - None, + /*auth_manager*/ None, Some(Ok(gateway)), )); let manager = Arc::new( - OpenAiModelsManager::new(home.path().to_path_buf(), endpoint, None).with_provider_catalog(), + OpenAiModelsManager::new( + home.path().to_path_buf(), + endpoint, + /*auth_manager*/ None, + ) + .with_provider_catalog(), ); - mount(&server, "old", 1111, Duration::from_millis(300)).await; + mount( + &server, + "old", + /*window*/ 1111, + Duration::from_millis(300), + ) + .await; let pending = tokio::spawn({ let manager = manager.clone(); async move { @@ -206,7 +221,7 @@ async fn changed_gateway_grant_during_catalog_response_cannot_publish_or_cache_o assert!(pending.await.unwrap().models.is_empty()); assert!(manager.get_remote_models().await.is_empty()); assert!(!home.path().join("models_cache.d").exists()); - mount(&server, "new", 2222, Duration::ZERO).await; + mount(&server, "new", /*window*/ 2222, Duration::ZERO).await; assert_eq!( manager .raw_model_catalog(RefreshStrategy::OnlineIfUncached, HTTP) @@ -224,12 +239,17 @@ async fn trusted_unauthenticated_catalog_url_is_discovered_through_the_models_ma info.gateway_oauth = None; Mock::given(method("GET")) .and(path("/models")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({"models": [model(1234)]}))) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"models": [model(/*window*/ 1234)]})), + ) .expect(1) .mount(&server) .await; - let endpoint = Arc::new(OpenAiModelsEndpoint::new(info, None, None)); - let manager = OpenAiModelsManager::new_without_cache(endpoint, None).with_provider_catalog(); + let endpoint = Arc::new(OpenAiModelsEndpoint::new( + info, /*auth_manager*/ None, /*gateway_auth_manager*/ None, + )); + let manager = OpenAiModelsManager::new_without_cache(endpoint, /*auth_manager*/ None) + .with_provider_catalog(); assert_eq!( manager .raw_model_catalog(RefreshStrategy::OnlineIfUncached, HTTP) @@ -260,10 +280,18 @@ async fn catalog_metadata_fails_closed_promptly_when_the_real_store_lock_is_busy &primary, json!({"access_token":"lock-test", "expires_at":i64::MAX}), ); - let endpoint = Arc::new(OpenAiModelsEndpoint::new(info, None, Some(Ok(gateway)))); - let manager = - OpenAiModelsManager::new(home.path().to_path_buf(), endpoint, None).with_provider_catalog(); - mount(&server, "lock-test", 1234, Duration::ZERO).await; + let endpoint = Arc::new(OpenAiModelsEndpoint::new( + info, + /*auth_manager*/ None, + Some(Ok(gateway)), + )); + let manager = OpenAiModelsManager::new( + home.path().to_path_buf(), + endpoint, + /*auth_manager*/ None, + ) + .with_provider_catalog(); + mount(&server, "lock-test", /*window*/ 1234, Duration::ZERO).await; assert_eq!( manager .raw_model_catalog(RefreshStrategy::Online, HTTP) @@ -362,7 +390,7 @@ async fn rejected_catalog_invalidates_the_sent_grant_after_a_preflight_account_s let endpoint = Arc::new(PausedGatewayEndpoint { inner: Arc::new(OpenAiModelsEndpoint::new( info.clone(), - None, + /*auth_manager*/ None, Some(Ok(gateway)), )), pause: std::sync::atomic::AtomicBool::new(false), @@ -370,10 +398,14 @@ async fn rejected_catalog_invalidates_the_sent_grant_after_a_preflight_account_s release: Default::default(), }); let manager = Arc::new( - OpenAiModelsManager::new(home.path().to_path_buf(), endpoint.clone(), None) - .with_provider_catalog(), + OpenAiModelsManager::new( + home.path().to_path_buf(), + endpoint.clone(), + /*auth_manager*/ None, + ) + .with_provider_catalog(), ); - mount(&server, "account-a", 1000, Duration::ZERO).await; + mount(&server, "account-a", /*window*/ 1000, Duration::ZERO).await; assert_eq!( manager .raw_model_catalog(RefreshStrategy::Online, HTTP) @@ -383,7 +415,7 @@ async fn rejected_catalog_invalidates_the_sent_grant_after_a_preflight_account_s Some(1000) ); let _b = seed_gateway_auth(&info, &primary, grant("account-b")); - mount(&server, "account-b", 2000, Duration::ZERO).await; + mount(&server, "account-b", /*window*/ 2000, Duration::ZERO).await; assert_eq!( manager .raw_model_catalog(RefreshStrategy::Online, HTTP) @@ -452,7 +484,7 @@ async fn rejected_catalog_invalidates_the_sent_grant_after_a_preflight_account_s let _b = seed_gateway_auth(&info, &primary, grant("account-b")); server.verify().await; server.reset().await; - mount(&server, "account-b", 3000, Duration::ZERO).await; + mount(&server, "account-b", /*window*/ 3000, Duration::ZERO).await; assert_eq!( manager .raw_model_catalog(RefreshStrategy::OnlineIfUncached, HTTP) diff --git a/codex-rs/models-manager/src/cache_identity_tests.rs b/codex-rs/models-manager/src/cache_identity_tests.rs index 3c084d22f9a6..9eb89ca7835b 100644 --- a/codex-rs/models-manager/src/cache_identity_tests.rs +++ b/codex-rs/models-manager/src/cache_identity_tests.rs @@ -40,7 +40,7 @@ impl ModelsEndpointClient for ObservedEndpoint { *self.persisted.lock().unwrap() = Some("grant-b".into()); } Ok(ModelsEndpointResponse { - models: vec![remote_model("shared", "Grant A", 0)], + models: vec![remote_model("shared", "Grant A", /*priority*/ 0)], etag: None, identity, }) @@ -104,8 +104,12 @@ async fn persisted_identity_changes_during_fetch_or_cache_write_cannot_publish_s replace_during_store: !during_fetch, entry: Mutex::new(None), }); - let manager = OpenAiModelsManager::new_with_cache(cache.clone(), endpoint.clone(), None) - .with_provider_catalog(); + let manager = OpenAiModelsManager::new_with_cache( + cache.clone(), + endpoint.clone(), + /*auth_manager*/ None, + ) + .with_provider_catalog(); assert!( manager .raw_model_catalog(RefreshStrategy::Online, DEFAULT_HTTP_CLIENT_FACTORY) @@ -141,7 +145,8 @@ async fn metadata_reads_observe_persisted_identity_changes_without_catalog_refre replace_during_fetch: false, }); let manager = - OpenAiModelsManager::new_without_cache(endpoint.clone(), None).with_provider_catalog(); + OpenAiModelsManager::new_without_cache(endpoint.clone(), /*auth_manager*/ None) + .with_provider_catalog(); assert_eq!( manager .raw_model_catalog(RefreshStrategy::Online, DEFAULT_HTTP_CLIENT_FACTORY) @@ -173,9 +178,9 @@ async fn partitioned_files_preserve_same_slug_metadata_for_each_provider_and_ten let path = home.path().join(MODEL_CACHE_FILE); let cache = FileModelsCache::partitioned(path.clone(), DEFAULT_MODEL_CACHE_TTL); let version = crate::client_version_to_whole(); - let mut first = remote_model("shared", "OpenAI", 0); + let mut first = remote_model("shared", "OpenAI", /*priority*/ 0); first.context_window = Some(128_000); - let mut second = remote_model("shared", "GLM", 0); + let mut second = remote_model("shared", "GLM", /*priority*/ 0); second.context_window = Some(32_000); second.default_reasoning_level = Some(ReasoningEffort::Low); let entry = |identity: &str, model| ModelsCacheEntry { @@ -334,14 +339,14 @@ impl ModelsEndpointClient for RejectedAttemptEndpoint { if self.reject.swap(false, Ordering::SeqCst) { // The actual rejected B request differs from preflight/current A. return Err(ModelsEndpointError { - error: catalog_http_error(403), + error: catalog_http_error(/*status*/ 403), identity: Some("grant-b".into()), }); } Ok(ModelsEndpointResponse { identity: self.identity().unwrap(), etag: None, - models: vec![remote_model("shared", "fresh", 0)], + models: vec![remote_model("shared", "fresh", /*priority*/ 0)], }) }) } @@ -350,7 +355,7 @@ impl ModelsEndpointClient for RejectedAttemptEndpoint { #[tokio::test] async fn rejection_survives_account_switches_when_persistent_invalidation_fails() { for load_error in [false, true] { - let stale = vec![remote_model("shared", "stale B", 0)]; + let stale = vec![remote_model("shared", "stale B", /*priority*/ 0)]; let cache = Arc::new(TestModelsCache { entry: Mutex::new(Some(ModelsCacheEntry { fetched_at: Utc::now(), @@ -367,8 +372,12 @@ async fn rejection_survives_account_switches_when_persistent_invalidation_fails( observed: Mutex::new("grant-a".into()), reject: std::sync::atomic::AtomicBool::new(true), }); - let manager = OpenAiModelsManager::new_with_cache(cache.clone(), endpoint.clone(), None) - .with_provider_catalog(); + let manager = OpenAiModelsManager::new_with_cache( + cache.clone(), + endpoint.clone(), + /*auth_manager*/ None, + ) + .with_provider_catalog(); assert!( manager .raw_model_catalog(RefreshStrategy::Online, DEFAULT_HTTP_CLIENT_FACTORY) From 23b948330211191ff986092e6061b85a483a63f1 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 22:50:27 -0400 Subject: [PATCH 22/41] fix(models): preserve existing async single-flight reconciliation (cherry picked from commit 45c2f27b665b3a16fb94aa7c4746b9b44cbaba10) --- .../model-provider/src/models_endpoint.rs | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/codex-rs/model-provider/src/models_endpoint.rs b/codex-rs/model-provider/src/models_endpoint.rs index 1ded3952268e..b7102a241418 100644 --- a/codex-rs/model-provider/src/models_endpoint.rs +++ b/codex-rs/model-provider/src/models_endpoint.rs @@ -56,7 +56,9 @@ pub(crate) struct OpenAiModelsEndpoint { gateway_auth_manager: Option, String>>, transport_builder: Arc, gateway_scope: RwLock>, - gateway_reconciliation: tokio::sync::Mutex<()>, + // A single-flight admission permit coordinates async observation; it does + // not guard data. Cached scope writes retain their short synchronous lock. + gateway_reconciliation: tokio::sync::Semaphore, } impl OpenAiModelsEndpoint { @@ -76,7 +78,7 @@ impl OpenAiModelsEndpoint { gateway_auth_manager, transport_builder: Arc::new(RouteAwareModelsTransportBuilder { redirect_policy }), gateway_scope: RwLock::new(None), - gateway_reconciliation: tokio::sync::Mutex::new(()), + gateway_reconciliation: tokio::sync::Semaphore::new(/*permits*/ 1), } } @@ -240,7 +242,13 @@ impl ModelsEndpointClient for OpenAiModelsEndpoint { if self.provider_info.gateway_oauth.is_none() { return; } - let _guard = self.gateway_reconciliation.lock().await; + let Ok(_admission) = self.gateway_reconciliation.acquire().await else { + *self + .gateway_scope + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = None; + return; + }; *self .gateway_scope .write() @@ -544,7 +552,7 @@ mod tests { let auth = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("test-api-key")); let endpoint = Arc::new(OpenAiModelsEndpoint { gateway_scope: RwLock::new(None), - gateway_reconciliation: tokio::sync::Mutex::new(()), + gateway_reconciliation: tokio::sync::Semaphore::new(/*permits*/ 1), provider_info: ModelProviderInfo { name: name.to_string(), ..ModelProviderInfo::create_openai_provider(base_url.map(str::to_string)) @@ -586,7 +594,7 @@ mod tests { let auth_manager = auth.map(AuthManager::from_auth_for_testing); let endpoint = Arc::new(OpenAiModelsEndpoint { gateway_scope: RwLock::new(None), - gateway_reconciliation: tokio::sync::Mutex::new(()), + gateway_reconciliation: tokio::sync::Semaphore::new(/*permits*/ 1), provider_info: ModelProviderInfo { experimental_bearer_token: Some("provider-key".into()), ..ModelProviderInfo::create_openai_provider(/*base_url*/ None) @@ -668,7 +676,7 @@ mod tests { let observed_request = Arc::new(Mutex::new(None)); let endpoint = OpenAiModelsEndpoint { gateway_scope: RwLock::new(None), - gateway_reconciliation: tokio::sync::Mutex::new(()), + gateway_reconciliation: tokio::sync::Semaphore::new(/*permits*/ 1), provider_info: ModelProviderInfo::create_openai_provider(Some(server.uri())), auth_manager: None, gateway_auth_manager: None, @@ -716,7 +724,7 @@ mod tests { )])); let endpoint = OpenAiModelsEndpoint { gateway_scope: RwLock::new(None), - gateway_reconciliation: tokio::sync::Mutex::new(()), + gateway_reconciliation: tokio::sync::Semaphore::new(/*permits*/ 1), provider_info, auth_manager: None, gateway_auth_manager: None, From b074eda906e5126715d94b3ed296f45147e7ba74 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 23:30:36 -0400 Subject: [PATCH 23/41] Reconcile existing embedded constructor onto accepted integration (Refs #54) (cherry picked from commit b471152a31205508974dbb3f72bab478d0bb48f4) --- codex-rs/Cargo.lock | 2 + codex-rs/core-api/src/lib.rs | 2 + codex-rs/core/src/config/embedded.rs | 130 ++++++++ codex-rs/core/src/config/embedded_tests.rs | 223 +++++++++++++ codex-rs/core/src/config/mod.rs | 2 + codex-rs/thread-manager-sample/Cargo.toml | 4 + codex-rs/thread-manager-sample/src/main.rs | 356 ++++++--------------- 7 files changed, 468 insertions(+), 251 deletions(-) create mode 100644 codex-rs/core/src/config/embedded.rs create mode 100644 codex-rs/core/src/config/embedded_tests.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 031fa98122f3..e7e5ee46b33c 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -4964,6 +4964,8 @@ dependencies = [ "clap", "codex-core-api", "serde_json", + "tempfile", + "tokio", "tracing", ] diff --git a/codex-rs/core-api/src/lib.rs b/codex-rs/core-api/src/lib.rs index 5c9fcdd84bb1..a9846a522f19 100644 --- a/codex-rs/core-api/src/lib.rs +++ b/codex-rs/core-api/src/lib.rs @@ -65,7 +65,9 @@ pub use codex_core::WaitForEnvironmentToolConfig; pub use codex_core::WithTurnExtensionData; pub use codex_core::build_models_manager; pub use codex_core::config::Config; +pub use codex_core::config::ConfigTomlLoadResult; pub use codex_core::config::Constrained; +pub use codex_core::config::EmbeddedHostConfigOptions; pub use codex_core::config::ExtraConfig; pub use codex_core::config::GhostSnapshotConfig; pub use codex_core::config::MultiAgentV2Config; diff --git a/codex-rs/core/src/config/embedded.rs b/codex-rs/core/src/config/embedded.rs new file mode 100644 index 000000000000..b47f04549705 --- /dev/null +++ b/codex-rs/core/src/config/embedded.rs @@ -0,0 +1,130 @@ +use super::Config; +use super::ConfigOverrides; +use super::ConfigTomlLoadResult; +use super::LOCAL_FS; +use super::bootstrap_auth_config; +use super::resolve_bootstrap_respect_system_proxy; +use codex_config::ConfigLayerStack; +use codex_config::ConfigRequirements; +use codex_config::ConfigRequirementsToml; +use codex_config::config_toml::ConfigToml; +use codex_config::types::AuthKeyringBackendKind; +use codex_config::types::SessionPickerViewMode; +use codex_features::Feature; +use codex_model_provider_info::OPENAI_PROVIDER_ID; +use codex_protocol::config_types::WebSearchMode; +use codex_protocol::models::PermissionProfile; +use codex_protocol::protocol::AskForApproval; +use codex_utils_absolute_path::AbsolutePathBuf; +use std::path::PathBuf; + +/// Host paths and model for an isolated, ephemeral OpenAI embedding. +/// +/// This supplies no delegated-child authority or prepared provider state. +pub struct EmbeddedHostConfigOptions { + pub codex_home: AbsolutePathBuf, + pub cwd: AbsolutePathBuf, + pub model: Option, + pub codex_self_exe: Option, + pub codex_linux_sandbox_exe: Option, +} + +impl Config { + /// Construct the sample host's read-only/Never configuration through the + /// canonical initializer, retaining only authentication from the bootstrap. + /// + /// Unlike the normal configuration builder, this intentionally does not + /// import bootstrap execution settings, providers, roles, plugins or hooks. + /// The bootstrap's validated authentication restrictions remain applicable + /// to every authentication manager subsequently built from this config. + pub async fn for_embedded_host( + options: EmbeddedHostConfigOptions, + auth_bootstrap: &ConfigTomlLoadResult, + ) -> std::io::Result { + let auth = bootstrap_auth_config(options.codex_home.as_path(), auth_bootstrap)?; + let requirements = auth_bootstrap.config_layer_stack.requirements(); + let requirements_toml = auth_bootstrap.config_layer_stack.requirements_toml(); + let stack = ConfigLayerStack::new( + Vec::new(), + ConfigRequirements { + allowed_login_methods: requirements.allowed_login_methods.clone(), + allowed_chatgpt_workspaces: requirements.allowed_chatgpt_workspaces.clone(), + ..Default::default() + }, + ConfigRequirementsToml { + allowed_login_methods: requirements_toml.allowed_login_methods.clone(), + allowed_chatgpt_workspaces: requirements_toml.allowed_chatgpt_workspaces.clone(), + ..Default::default() + }, + )?; + let cfg = ConfigToml { + // Explicit state roots prevent ambient CODEX_SQLITE_HOME selection. + sqlite_home: Some(options.codex_home.clone()), + mcp_optional_startup_grace_ms: Some(1000), + include_permissions_instructions: Some(false), + include_apps_instructions: Some(false), + include_collaboration_mode_instructions: Some(false), + include_environment_context: Some(false), + web_search: Some(WebSearchMode::Disabled), + ..Default::default() + }; + let overrides = ConfigOverrides { + model: options.model, + cwd: Some(options.cwd.to_path_buf()), + model_provider: Some(OPENAI_PROVIDER_ID.to_string()), + approval_policy: Some(AskForApproval::Never), + permission_profile: Some(PermissionProfile::read_only()), + ephemeral: Some(true), + codex_self_exe: options.codex_self_exe, + codex_linux_sandbox_exe: options.codex_linux_sandbox_exe, + ..Default::default() + }; + let mut config = Self::load_config_with_layer_stack( + LOCAL_FS.as_ref(), + cfg, + overrides, + options.codex_home, + stack, + ) + .await?; + // Preserve the sample's explicit deviations from full-loader defaults. + config.include_skill_instructions = false; + config.cloud_skill_enabled = false; + config.orchestrator_mcp_enabled = false; + config.agent_max_threads = Some(6); + config.agent_max_depth = 1; + config.agent_interrupt_message_enabled = false; + config.tui_fullscreen_transcript = false; + config.tui_session_picker_view = SessionPickerViewMode::Dense; + config.check_for_update_on_startup = false; + config.analytics_enabled = Some(false); + config.feedback_enabled = false; + config.update_plan_enabled = true; + // The ordinary loader coerces secure stores to File for version0.0.0. + // Embedded cargo-run hosts retain their validated bootstrap selection. + config.cli_auth_credentials_store_mode = auth.auth_credentials_store_mode; + config.mcp_oauth_credentials_store_mode = auth_bootstrap + .config_toml + .mcp_oauth_credentials_store + .unwrap_or_default(); + config.application_auth_route_config = Some(auth.auth_route_config.clone()); + config.forced_login_method = auth.forced_login_method; + config.forced_chatgpt_workspace_id = auth.forced_chatgpt_workspace_id; + config.chatgpt_base_url = auth + .chatgpt_base_url + .unwrap_or_else(|| "https://chatgpt.com/backend-api/".to_string()); + config.respect_system_proxy = resolve_bootstrap_respect_system_proxy( + &auth_bootstrap.config_toml, + requirements.feature_requirements.as_ref(), + )?; + config.features.set_enabled( + Feature::SecretAuthStorage, + auth.keyring_backend_kind == AuthKeyringBackendKind::Secrets, + )?; + Ok(config) + } +} + +#[cfg(test)] +#[path = "embedded_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/config/embedded_tests.rs b/codex-rs/core/src/config/embedded_tests.rs new file mode 100644 index 000000000000..d050c978a023 --- /dev/null +++ b/codex-rs/core/src/config/embedded_tests.rs @@ -0,0 +1,223 @@ +use super::*; +use codex_config::RequirementSource; +use codex_config::Sourced; +use codex_config::config_toml::ForcedChatgptWorkspaceIds; +use codex_config::types::AuthCredentialsStoreMode; +use codex_config::types::OAuthCredentialsStoreMode; +use codex_features::Features; +use codex_protocol::config_types::ForcedLoginMethod; +use pretty_assertions::assert_eq; + +fn options(home: &std::path::Path, cwd: &std::path::Path) -> EmbeddedHostConfigOptions { + EmbeddedHostConfigOptions { + codex_home: AbsolutePathBuf::from_absolute_path(home).expect("absolute home"), + cwd: AbsolutePathBuf::from_absolute_path(cwd).expect("absolute cwd"), + model: Some("host-model".to_string()), + codex_self_exe: Some(cwd.join("host-codex")), + codex_linux_sandbox_exe: Some(cwd.join("host-sandbox")), + } +} + +#[tokio::test] +async fn bootstrap_execution_layers_are_not_imported() -> std::io::Result<()> { + let home = tempfile::tempdir()?; + let cwd = tempfile::tempdir()?; + std::fs::write( + home.path().join("config.toml"), + r#" +model = "foreign-model" +model_provider = "foreign" +openai_base_url = "https://foreign.invalid/v1" +approval_policy = "never" +sandbox_mode = "danger-full-access" +instructions = "foreign instructions" +developer_instructions = "foreign developer instructions" +notify = ["foreign-program"] +web_search = "live" +cli_auth_credentials_store = "keyring" +mcp_oauth_credentials_store = "auto" +[tools.update_plan] +enabled = false +[model_providers.foreign] +name = "Foreign" +base_url = "https://foreign.invalid/v1" +wire_api = "responses" +[mcp_servers.foreign] +command = "foreign-program" +[plugins."foreign@test"] +enabled = true +[[hooks.PreToolUse]] +matcher = "Bash" +[[hooks.PreToolUse.hooks]] +type = "command" +command = "foreign-hook-program" +[features] +secret_auth_storage = true +respect_system_proxy = true +"#, + )?; + let cwd_abs = AbsolutePathBuf::from_absolute_path(cwd.path())?; + let bootstrap = super::super::load_config_toml_with_layer_stack( + home.path(), + Some(&cwd_abs), + Vec::new(), + super::super::ConfigLoadOptions::default(), + ) + .await?; + assert!( + bootstrap + .config_layer_stack + .layers_low_to_high() + .next() + .is_some() + ); + let config = Config::for_embedded_host(options(home.path(), cwd.path()), &bootstrap).await?; + assert_eq!(config.model.as_deref(), Some("host-model")); + assert_eq!(config.model_provider_id, OPENAI_PROVIDER_ID); + assert!(!config.model_providers.contains_key("foreign")); + assert_eq!(config.model_provider.base_url, None); + assert_eq!( + config.permissions.approval_policy.value(), + AskForApproval::Never + ); + assert_eq!( + config.permissions.permission_profile(), + &PermissionProfile::read_only() + ); + assert_eq!(config.web_search_mode.value(), WebSearchMode::Disabled); + assert!(config.ephemeral); + assert!( + config + .config_layer_stack + .layers_low_to_high() + .next() + .is_none() + ); + assert!(config.mcp_servers.get().is_empty()); + assert!(config.agent_roles.is_empty()); + assert!(config.plugins.plugins.is_empty()); + assert!(config.respect_system_proxy); + assert!(config.base_instructions.is_none()); + assert!(config.developer_instructions.is_none()); + assert!(config.notify.is_none()); + assert!(config.prepared_child_models.is_none()); + assert!(!config.child_role_service_tier_override); + assert_eq!( + config.cli_auth_credentials_store_mode, + AuthCredentialsStoreMode::Keyring + ); + assert_eq!( + config.mcp_oauth_credentials_store_mode, + OAuthCredentialsStoreMode::Auto + ); + assert!(config.features.enabled(Feature::SecretAuthStorage)); + let mut expected_features = Features::with_defaults(); + expected_features.set_enabled(Feature::SecretAuthStorage, /*enabled*/ true); + assert_eq!(config.features.get(), &expected_features); + assert_eq!(config.sqlite.home(), home.path()); + assert_eq!(config.log_dir, home.path().join("log")); + assert_eq!( + config.sqlite.logs_db_path(), + codex_state::SqliteConfig::from_sqlite_home(config.codex_home.clone()).logs_db_path() + ); + assert!(config.update_plan_enabled); + assert!(!config.include_permissions_instructions); + assert!(!config.include_skill_instructions); + assert!(!config.cloud_skill_enabled); + assert!(!config.orchestrator_mcp_enabled); + assert_eq!(config.agent_max_threads, Some(6)); + assert_eq!(config.agent_max_depth, 1); + assert_eq!(config.analytics_enabled, Some(false)); + assert!(!config.feedback_enabled); + assert_eq!(config.codex_self_exe, Some(cwd.path().join("host-codex"))); + Ok(()) +} + +#[tokio::test] +async fn managed_auth_survives_rebuilding_from_embedded_config() -> std::io::Result<()> { + let home = tempfile::tempdir()?; + let cwd = tempfile::tempdir()?; + for store in [ + AuthCredentialsStoreMode::Auto, + AuthCredentialsStoreMode::Keyring, + ] { + let requirements = ConfigRequirements { + allowed_login_methods: Some(Sourced::new( + vec![ForcedLoginMethod::Chatgpt], + RequirementSource::Unknown, + )), + allowed_chatgpt_workspaces: Some(Sourced::new( + vec!["allowed".to_string()], + RequirementSource::Unknown, + )), + cli_auth_credentials_store: Some(Sourced::new(store, RequirementSource::Unknown)), + chatgpt_base_url: Some(Sourced::new( + "https://managed.invalid/backend-api/".to_string(), + RequirementSource::Unknown, + )), + ..Default::default() + }; + let bootstrap = ConfigTomlLoadResult { + config_toml: ConfigToml { + cli_auth_credentials_store: Some(AuthCredentialsStoreMode::File), + forced_chatgpt_workspace_id: Some(ForcedChatgptWorkspaceIds::Multiple(vec![ + "denied".to_string(), + " allowed ".to_string(), + ])), + ..Default::default() + }, + config_layer_stack: ConfigLayerStack::new( + Vec::new(), + requirements, + ConfigRequirementsToml::default(), + )?, + }; + let expected = bootstrap_auth_config(home.path(), &bootstrap)?; + let config = + Config::for_embedded_host(options(home.path(), cwd.path()), &bootstrap).await?; + let actual = config.auth_config(); + assert_eq!(actual.auth_credentials_store_mode, store); + assert_eq!(actual.chatgpt_base_url, expected.chatgpt_base_url); + assert_eq!( + actual.effective_chatgpt_workspaces(), + Some(vec!["allowed".to_string()]) + ); + assert!(actual.is_login_method_allowed(ForcedLoginMethod::Chatgpt)); + assert!(!actual.is_login_method_allowed(ForcedLoginMethod::Api)); + assert_eq!(actual.keyring_backend_kind, expected.keyring_backend_kind); + assert_eq!( + config.features.enabled(Feature::SecretAuthStorage), + Features::with_defaults().enabled(Feature::SecretAuthStorage) + ); + } + Ok(()) +} + +#[tokio::test] +async fn invalid_bootstrap_auth_is_rejected_before_host_construction() -> std::io::Result<()> { + let home = tempfile::tempdir()?; + let cwd = tempfile::tempdir()?; + let bootstrap = ConfigTomlLoadResult { + config_toml: ConfigToml::default(), + config_layer_stack: ConfigLayerStack::new( + Vec::new(), + ConfigRequirements { + allowed_login_methods: Some(Sourced::new( + vec![ForcedLoginMethod::Chatgpt], + RequirementSource::Unknown, + )), + allowed_chatgpt_workspaces: Some(Sourced::new( + Vec::new(), + RequirementSource::Unknown, + )), + ..Default::default() + }, + ConfigRequirementsToml::default(), + )?, + }; + let error = Config::for_embedded_host(options(home.path(), cwd.path()), &bootstrap) + .await + .expect_err("invalid managed authentication must fail"); + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + Ok(()) +} diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index a472d105c68d..cd0bda34ff18 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -163,6 +163,8 @@ use toml_edit::DocumentMut; mod auth_keyring; pub mod edit; +mod embedded; +pub use embedded::EmbeddedHostConfigOptions; mod managed_features; mod metrics; mod network_config; diff --git a/codex-rs/thread-manager-sample/Cargo.toml b/codex-rs/thread-manager-sample/Cargo.toml index 9e857ad0bc7f..89d519ae9f24 100644 --- a/codex-rs/thread-manager-sample/Cargo.toml +++ b/codex-rs/thread-manager-sample/Cargo.toml @@ -16,3 +16,7 @@ serde_json = { workspace = true } # additional `codex-*` crates here. codex-core-api = { workspace = true } tracing = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/codex-rs/thread-manager-sample/src/main.rs b/codex-rs/thread-manager-sample/src/main.rs index af50bd5f44f4..cd80472ae85e 100644 --- a/codex-rs/thread-manager-sample/src/main.rs +++ b/codex-rs/thread-manager-sample/src/main.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; -use std::collections::HashMap; use std::io::IsTerminal; use std::io::Read; use std::io::Write; @@ -9,62 +7,26 @@ use anyhow::Context; use anyhow::bail; use clap::Parser; use codex_core_api::AbsolutePathBuf; -use codex_core_api::AltScreenMode; -use codex_core_api::ApprovalsReviewer; use codex_core_api::Arg0DispatchPaths; -use codex_core_api::AskForApproval; -use codex_core_api::AuthKeyringBackendKind; use codex_core_api::AuthManager; -use codex_core_api::AutoCompactTokenLimitScope; use codex_core_api::CodexAppsToolsCache; use codex_core_api::CodexHomeUserInstructionsProvider; use codex_core_api::CodexThread; use codex_core_api::Config; -use codex_core_api::ConfigLayerStack; use codex_core_api::ConfigLoadOptions; -use codex_core_api::ConfigRequirements; -use codex_core_api::ConfigRequirementsToml; -use codex_core_api::Constrained; use codex_core_api::EnvironmentManager; use codex_core_api::EventMsg; use codex_core_api::ExecServerRuntimeOptions; use codex_core_api::ExtensionRegistryBuilder; -use codex_core_api::Feature; -use codex_core_api::Features; -use codex_core_api::GhostSnapshotConfig; -use codex_core_api::History; -use codex_core_api::MemoriesConfig; -use codex_core_api::ModelAvailabilityNuxConfig; -use codex_core_api::MultiAgentV2Config; use codex_core_api::NewThread; -use codex_core_api::Notice; -use codex_core_api::OPENAI_PROVIDER_ID; -use codex_core_api::OtelConfig; -use codex_core_api::PermissionProfile; -use codex_core_api::Permissions; -use codex_core_api::ProjectConfig; -use codex_core_api::RealtimeAudioConfig; -use codex_core_api::RealtimeConfig; -use codex_core_api::SessionPickerViewMode; use codex_core_api::SessionSource; -use codex_core_api::SqliteConfig; use codex_core_api::StartIfIdleSubmission; use codex_core_api::StartThreadOptions; -use codex_core_api::TerminalResizeReflowConfig; use codex_core_api::ThreadManager; -use codex_core_api::ThreadStoreConfig; -use codex_core_api::ToolSuggestConfig; -use codex_core_api::TuiKeymap; -use codex_core_api::TuiNotificationSettings; -use codex_core_api::TuiPetAnchor; use codex_core_api::TurnInputRequest; -use codex_core_api::UriBasedFileOpener; use codex_core_api::UserInput; -use codex_core_api::WebSearchMode; use codex_core_api::arg0_dispatch_or_else; -use codex_core_api::bootstrap_auth_config; use codex_core_api::build_models_manager; -use codex_core_api::built_in_model_providers; use codex_core_api::find_codex_home; use codex_core_api::init_state_db; use codex_core_api::install_image_generation_extension; @@ -72,7 +34,6 @@ use codex_core_api::item_event_to_server_notification; use codex_core_api::load_config_toml_with_layer_stack; use codex_core_api::local_agent_graph_store_from_state_db; use codex_core_api::passthrough_image_store; -use codex_core_api::resolve_bootstrap_respect_system_proxy; use codex_core_api::resolve_installation_id; use codex_core_api::set_default_originator; use codex_core_api::thread_store_from_config; @@ -188,6 +149,21 @@ async fn new_config( ) -> anyhow::Result { let codex_home = find_codex_home().context("find Codex home")?; let cwd = AbsolutePathBuf::current_dir().context("resolve current directory")?; + new_config_for_paths( + model, + arg0_paths, + AbsolutePathBuf::from_absolute_path(codex_home)?, + cwd, + ) + .await +} + +async fn new_config_for_paths( + model: Option, + arg0_paths: Arg0DispatchPaths, + codex_home: AbsolutePathBuf, + cwd: AbsolutePathBuf, +) -> anyhow::Result { // Resolve auth through the shared bootstrap path so cargo-run builds retain the configured // secure store instead of the full Config loader's local-development file-store override. let bootstrap = load_config_toml_with_layer_stack( @@ -198,219 +174,18 @@ async fn new_config( ) .await .context("load authentication configuration")?; - let auth_config = bootstrap_auth_config(codex_home.as_path(), &bootstrap)?; - let requirements = bootstrap.config_layer_stack.requirements(); - let requirements_toml = bootstrap.config_layer_stack.requirements_toml(); - // Preserve auth restrictions for every manager rebuilt from Config without importing - // execution settings, plugins, or hooks from the bootstrap configuration. - let config_layer_stack = ConfigLayerStack::new( - Vec::new(), - ConfigRequirements { - allowed_login_methods: requirements.allowed_login_methods.clone(), - allowed_chatgpt_workspaces: requirements.allowed_chatgpt_workspaces.clone(), - ..Default::default() + Config::for_embedded_host( + codex_core_api::EmbeddedHostConfigOptions { + codex_home, + cwd, + model, + codex_self_exe: arg0_paths.codex_self_exe, + codex_linux_sandbox_exe: arg0_paths.codex_linux_sandbox_exe, }, - ConfigRequirementsToml { - allowed_login_methods: requirements_toml.allowed_login_methods.clone(), - allowed_chatgpt_workspaces: requirements_toml.allowed_chatgpt_workspaces.clone(), - ..Default::default() - }, - )?; - let mcp_oauth_credentials_store_mode = bootstrap - .config_toml - .mcp_oauth_credentials_store - .unwrap_or_default(); - let respect_system_proxy = resolve_bootstrap_respect_system_proxy( - &bootstrap.config_toml, - bootstrap - .config_layer_stack - .requirements() - .feature_requirements - .as_ref(), - )?; - let model_provider_id = OPENAI_PROVIDER_ID.to_string(); - let model_providers = built_in_model_providers(/*openai_base_url*/ None); - let model_provider = model_providers - .get(&model_provider_id) - .context("OpenAI model provider should be available")? - .clone(); - - let mut config = Config { - application_network_policy: Default::default(), - application_auth_route_config: Some(auth_config.auth_route_config.clone()), - config_layer_stack, - plugins: Default::default(), - startup_warnings: Vec::new(), - bypass_hook_trust: false, - model, - daybreak_enabled: false, - service_tier: None, - review_model: None, - model_context_window: None, - model_auto_compact_token_limit: None, - model_auto_compact_token_limit_scope: AutoCompactTokenLimitScope::Total, - model_post_turn_compact_threshold_percent: 0, - async_compaction_threshold_percent: 65, - model_provider_id, - model_provider, - personality: None, - permissions: Permissions::from_approval_and_profile( - Constrained::allow_any(AskForApproval::Never), - Constrained::allow_any(PermissionProfile::read_only()), - )?, - explicit_permission_profile_mode: false, - custom_permission_profiles: Vec::new(), - approvals_reviewer: ApprovalsReviewer::User, - enforce_residency: Constrained::allow_any(/*initial_value*/ None), - hide_agent_reasoning: false, - show_raw_agent_reasoning: false, - base_instructions: None, - base_instructions_provenance: None, - developer_instructions: None, - guardian_policy_config: None, - guardian_extra_policy: None, - guardian_policy_template: None, - guardian_transcript_mode: Default::default(), - guardian_conversation_history_prompt: None, - guardian_conversation_history_max_output_tokens: None, - guardian_circuit_break_action: Default::default(), - include_permissions_instructions: false, - include_apps_instructions: false, - include_collaboration_mode_instructions: false, - include_skill_instructions: false, - skill_max_context_tokens: None, - cloud_skill_enabled: false, - orchestrator_mcp_enabled: false, - include_environment_context: false, - include_environment_context_time: true, - compact_prompt: None, - notify: None, - tui_notifications: TuiNotificationSettings::default(), - animations: true, - tui_effects: Default::default(), - tui_rendering: Default::default(), - show_tooltips: true, - tui_show_server_version_notice: true, - tui_auto_recap: true, - model_availability_nux: ModelAvailabilityNuxConfig::default(), - tui_fullscreen_transcript: false, - tui_mouse_scroll_speed: None, - tui_copy_on_select: Default::default(), - tui_copy_on_select_clear_selection: false, - tui_right_click_paste: Default::default(), - tui_alternate_screen: AltScreenMode::Auto, - tui_status_line: None, - tui_status_line_use_colors: true, - tui_terminal_title: None, - tui_theme: None, - tui_raw_output_mode: false, - tui_pet: None, - tui_pet_anchor: TuiPetAnchor::Composer, - terminal_resize_reflow: TerminalResizeReflowConfig::default(), - tui_keymap: TuiKeymap::default(), - tui_quick_dials: BTreeMap::new(), - tui_session_picker_view: SessionPickerViewMode::Dense, - tui_agents_overview_grouping: Default::default(), - tui_resume_cwd: None, - tui_vim_mode_default: false, - tui_question_esc_back: true, - cwd: cwd.clone(), - workspace_roots: vec![cwd], - workspace_roots_explicit: false, - cli_auth_credentials_store_mode: auth_config.auth_credentials_store_mode, - mcp_servers: Constrained::allow_any(HashMap::new()), - mcp_enterprise_managed_auth: None, - non_prefixed_mcp_tool_servers: None, - mcp_oauth_credentials_store_mode, - mcp_oauth_callback_port: None, - mcp_oauth_callback_url: None, - mcp_optional_startup_grace: std::time::Duration::from_secs(1), - model_providers, - project_doc_max_bytes: 32 * 1024, - project_doc_fallback_filenames: Vec::new(), - tool_output_token_limit: None, - agents_enabled: true, - agent_max_threads: Some(6), - agent_default_subagent_model: None, - agent_default_subagent_reasoning_effort: None, - agent_interrupt_message_enabled: false, - agent_max_depth: 1, - agent_roles: BTreeMap::new(), - memories: MemoriesConfig::default(), - sqlite: SqliteConfig::from_sqlite_home(codex_home.clone()), - log_dir: codex_home.join("log").to_path_buf(), - codex_home, - history: History::default(), - ephemeral: true, - extra_config: None, - file_opener: UriBasedFileOpener::VsCode, - codex_self_exe: arg0_paths.codex_self_exe, - codex_linux_sandbox_exe: arg0_paths.codex_linux_sandbox_exe, - main_execve_wrapper_exe: None, - zsh_path: None, - model_reasoning_effort: None, - plan_mode_reasoning_effort: None, - model_reasoning_summary: None, - model_catalog: None, - model_verbosity: None, - chatgpt_base_url: auth_config - .chatgpt_base_url - .clone() - .unwrap_or_else(|| "https://chatgpt.com/backend-api/".to_string()), - respect_system_proxy, - apps_mcp_product_sku: None, - responses_api_metadata: BTreeMap::new(), - realtime_audio: RealtimeAudioConfig::default(), - experimental_realtime_ws_base_url: None, - experimental_realtime_webrtc_call_base_url: None, - experimental_realtime_ws_model: None, - realtime: RealtimeConfig::default(), - experimental_realtime_ws_backend_prompt: None, - experimental_realtime_ws_startup_context: None, - experimental_realtime_start_instructions: None, - experimental_thread_store: ThreadStoreConfig::Local, - forced_chatgpt_workspace_id: auth_config.forced_chatgpt_workspace_id.clone(), - forced_login_method: auth_config.forced_login_method, - web_search_mode: Constrained::allow_any(WebSearchMode::Disabled), - web_search_config: None, - experimental_request_user_input_enabled: true, - update_plan_enabled: true, - tool_registry: Default::default(), - code_mode: Default::default(), - background_terminal_max_timeout: 300_000, - thread_unload_delay: std::time::Duration::from_secs(/*secs*/ 1800), - ghost_snapshot: GhostSnapshotConfig::default(), - multi_agent_v2: MultiAgentV2Config::default(), - max_goal_token_budget: None, - token_budget: None, - token_budget_startup_config: None, - rollout_budget: None, - current_time_reminder: None, - sleep_tool_mode: Default::default(), - features: Default::default(), - prefer_mxc: false, - - runtime_feature_defaults: Default::default(), - suppress_unstable_features_warning: false, - active_project: ProjectConfig { trust_level: None }, - notices: Notice::default(), - check_for_update_on_startup: false, - disable_paste_burst: false, - analytics_enabled: Some(false), - feedback_enabled: false, - tool_suggest: ToolSuggestConfig::default(), - otel: OtelConfig::default(), - }; - config - .features - .set(Features::with_defaults()) - .context("configure default features")?; - // AuthManager and MCP both derive their credential backend from Config. - config.features.set_enabled( - Feature::SecretAuthStorage, - auth_config.keyring_backend_kind == AuthKeyringBackendKind::Secrets, - )?; - Ok(config) + &bootstrap, + ) + .await + .context("construct isolated embedded configuration") } async fn run_turn(thread: &CodexThread, thread_id: &str, prompt: String) -> anyhow::Result<()> { @@ -507,3 +282,82 @@ async fn run_turn(thread: &CodexThread, thread_id: &str, prompt: String) -> anyh } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn public_facade_constructs_isolated_host_with_bootstrap_stores() -> anyhow::Result<()> { + let home = tempfile::tempdir()?; + let cwd = tempfile::tempdir()?; + std::fs::write( + home.path().join("config.toml"), + r#" +model = "foreign-model" +openai_base_url = "https://foreign.invalid/v1" +sandbox_mode = "danger-full-access" +cli_auth_credentials_store = "keyring" +mcp_oauth_credentials_store = "auto" +notify = ["foreign-program"] +[tools.update_plan] +enabled = false +[features] +secret_auth_storage = true +[mcp_servers.foreign] +command = "foreign-program" +"#, + )?; + let config = new_config_for_paths( + Some("host-model".to_string()), + Arg0DispatchPaths::default(), + AbsolutePathBuf::from_absolute_path(home.path())?, + AbsolutePathBuf::from_absolute_path(cwd.path())?, + ) + .await?; + assert_eq!(config.model.as_deref(), Some("host-model")); + assert_eq!(config.model_provider_id, "openai"); + assert_eq!(config.model_provider.base_url, None); + assert!(config.ephemeral); + assert_eq!( + config.permissions.approval_policy.value(), + codex_core_api::AskForApproval::Never + ); + assert_eq!( + config.permissions.permission_profile(), + &codex_core_api::PermissionProfile::read_only() + ); + assert!(config.mcp_servers.get().is_empty()); + assert!(config.agent_roles.is_empty()); + assert!(config.notify.is_none()); + assert!( + config + .config_layer_stack + .layers_low_to_high() + .next() + .is_none() + ); + assert_eq!(config.sqlite.home(), home.path()); + assert_eq!(config.log_dir, home.path().join("log")); + assert_eq!( + config.sqlite.logs_db_path(), + codex_core_api::SqliteConfig::from_sqlite_home(config.codex_home.clone()) + .logs_db_path() + ); + assert!(config.update_plan_enabled); + assert_eq!( + config.cli_auth_credentials_store_mode, + codex_core_api::AuthCredentialsStoreMode::Keyring + ); + assert_eq!( + config.mcp_oauth_credentials_store_mode, + codex_core_api::OAuthCredentialsStoreMode::Auto + ); + assert!( + config + .features + .enabled(codex_core_api::Feature::SecretAuthStorage) + ); + Ok(()) + } +} From 907b86d9f37dded646bc13cad48b2c2455d07583 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 01:06:31 -0400 Subject: [PATCH 24/41] test(secrets): move consumed ciphertext in authentication fixture (Refs #50) (cherry picked from commit af4324e2973979e6c717a23ce8e670155d3e6980) --- codex-rs/secrets/src/local.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex-rs/secrets/src/local.rs b/codex-rs/secrets/src/local.rs index 727c4cbdfe28..50dcc3840587 100644 --- a/codex-rs/secrets/src/local.rs +++ b/codex-rs/secrets/src/local.rs @@ -464,7 +464,7 @@ mod tests { ] ); assert!(decrypt_with_passphrase(&ciphertext, &"wrong-password".to_owned().into()).is_err()); - let mut damaged = ciphertext.clone(); + let mut damaged = ciphertext; *damaged.last_mut().expect("ciphertext payload") ^= 1; assert!(decrypt_with_passphrase(&damaged, &passphrase).is_err()); let fresh = From 537e53a670ca7e9fde863a9da4ae3dae3b1f8fea Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 09:49:18 -0400 Subject: [PATCH 25/41] Preserve accepted SDK LF generation while composing capacity qualification (Refs #86) --- sdk/python/scripts/update_sdk_artifacts.py | 10 ++++++++ .../test_artifact_workflow_and_binaries.py | 25 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/sdk/python/scripts/update_sdk_artifacts.py b/sdk/python/scripts/update_sdk_artifacts.py index bc32e7ce04dc..cc260117a329 100755 --- a/sdk/python/scripts/update_sdk_artifacts.py +++ b/sdk/python/scripts/update_sdk_artifacts.py @@ -1426,6 +1426,16 @@ def generate_types_from_schema_dir(schema_dir: Path, *, format_output: bool = Tr generate_v2_all(schema_dir, format_output=format_output) generate_notification_registry(schema_dir) generate_public_api_flat_methods(format_output=format_output) + # Generated Python artifacts have the same LF byte contract on every platform. + for relative_path in ( + "src/openai_codex/generated/v2_all.py", + "src/openai_codex/generated/notification_registry.py", + "src/openai_codex/api.py", + ): + path = sdk_root() / relative_path + if relative_path == "src/openai_codex/api.py" and not path.exists(): + continue + path.write_text(path.read_text(encoding="utf-8"), encoding="utf-8", newline="\n") def build_parser() -> argparse.ArgumentParser: diff --git a/sdk/python/tests/test_artifact_workflow_and_binaries.py b/sdk/python/tests/test_artifact_workflow_and_binaries.py index e78493880d24..4a1c5338dff9 100644 --- a/sdk/python/tests/test_artifact_workflow_and_binaries.py +++ b/sdk/python/tests/test_artifact_workflow_and_binaries.py @@ -390,6 +390,31 @@ def test_generate_types_wires_all_generation_steps() -> None: ] +def test_generate_types_preserves_optional_missing_public_api( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + script = _load_update_script_module() + monkeypatch.setattr(script, "sdk_root", lambda: tmp_path) + generated_dir = tmp_path / "src" / "openai_codex" / "generated" + generated_dir.mkdir(parents=True) + content = "# café\r\n".encode("utf-8") + + def generate_v2_all(schema_dir: Path, *, format_output: bool) -> None: + (generated_dir / "v2_all.py").write_bytes(content) + + def generate_notification_registry(schema_dir: Path) -> None: + (generated_dir / "notification_registry.py").write_bytes(content) + + monkeypatch.setattr(script, "generate_v2_all", generate_v2_all) + monkeypatch.setattr(script, "generate_notification_registry", generate_notification_registry) + + script.generate_types_from_schema_dir(tmp_path / "schema", format_output=False) + + for name in ("v2_all.py", "notification_registry.py"): + assert (generated_dir / name).read_bytes() == "# café\n".encode("utf-8") + assert not (tmp_path / "src" / "openai_codex" / "api.py").exists() + + @pytest.mark.parametrize("schema_override", [None, "override-schema"]) def test_generation_resolves_configured_schema_and_explicit_override( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, schema_override: str | None From e8af1492ca5d2bd402dd3666e51261d20751a865 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 10:48:32 -0400 Subject: [PATCH 26/41] test: keep hosted timeout and VHD fixtures portable --- .../scripts/test_ci_checkout_and_lint_policy.py | 1 + .../test_windows_ci_resource_observation.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/scripts/test_ci_checkout_and_lint_policy.py b/.github/scripts/test_ci_checkout_and_lint_policy.py index 43393abefdd5..8d731e7a0a1c 100644 --- a/.github/scripts/test_ci_checkout_and_lint_policy.py +++ b/.github/scripts/test_ci_checkout_and_lint_policy.py @@ -16,6 +16,7 @@ def test_fork_cold_build_budgets_preserve_upstream_and_short_jobs(self): "test": ("30", 90), "test-windows-shard": ("30", 90), "test-windows-native-main": ("40", None), + "voice-runtime-diagnostic": ("60", None), "clippy": ("30", 90), "verify-release-build": ("30", 90), }, diff --git a/.github/scripts/test_windows_ci_resource_observation.py b/.github/scripts/test_windows_ci_resource_observation.py index 7703981cbcfe..0245f5c58169 100644 --- a/.github/scripts/test_windows_ci_resource_observation.py +++ b/.github/scripts/test_windows_ci_resource_observation.py @@ -406,19 +406,24 @@ def run_ownership_fixture( } function global:Test-Path { param($LiteralPath, $PathType) - if ($LiteralPath -ne 'C:\fixture\codex-dev-drive.vhdx') { throw 'unexpected VHD path' } + if ($LiteralPath -ne $global:fixtureVhd) { throw 'unexpected VHD path' } return $true } function global:Get-Item { param($LiteralPath) - if ($LiteralPath -ne 'C:\fixture\codex-dev-drive.vhdx') { throw 'unexpected VHD path' } + if ($LiteralPath -ne $global:fixtureVhd) { throw 'unexpected VHD path' } [pscustomobject]@{ Length=4096 } } -$env:RUNNER_TEMP = 'C:\fixture' +$env:RUNNER_TEMP = 'FIXTURE_ROOT' +$global:fixtureVhd = Join-Path $env:RUNNER_TEMP 'codex-dev-drive.vhdx' $epoch = [datetime]::new(1601, 1, 1, 0, 0, 0, [DateTimeKind]::Utc).Ticks $micro = [long](($global:fixtureCreated.Ticks - $epoch) / 10D) + OFFSET & $args[0] -OwnedPid 101 -CreationMicroseconds $micro """ + # Keep Windows C: semantics there; Linux PowerShell needs a native + # filesystem root for Join-Path and System.IO.Path root interpretation. + fixture_root = "C:\\fixture" if os.name == "nt" else tempfile.gettempdir() + fixture = fixture.replace("FIXTURE_ROOT", fixture_root.replace("'", "''")) fixture = fixture.replace("CHILDREN", str(children)).replace( "OFFSET", str(offset) ) @@ -452,7 +457,10 @@ def test_powershell_exact_root_identity_and_numeric_vhd_fixture(self): ) self.assertEqual(data["vhd"]["file_length_bytes"], 4096) self.assertEqual(data["vhd"]["logical_size_bytes"], 68719476736) - self.assertEqual(data["vhd"]["backing_volume"], "C:\\".replace("\\\\", "\\")) + self.assertEqual( + data["vhd"]["backing_volume"], + "C:\\" if os.name == "nt" else Path(tempfile.gettempdir()).anchor, + ) self.assertIsNone(data["vhd"]["allocated_physical_bytes"]) self.assertLessEqual(len(data["volumes"]), 16) From 4cc00fc341af12ada1a6c70092a0761ec535b01e Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 10:50:10 -0400 Subject: [PATCH 27/41] build: materialize Opus install headers inside output tree --- MODULE.bazel | 2 ++ patches/BUILD.bazel | 1 + ...usic-sys_materialize_install_headers.patch | 21 +++++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 patches/opusic-sys_materialize_install_headers.patch diff --git a/MODULE.bazel b/MODULE.bazel index 92c8e81cd961..ca1de5dc75a8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -666,6 +666,8 @@ crate.annotation( "@rules_foreign_cc//toolchains:current_ninja_toolchain", ], crate = "opusic-sys", + patch_args = ["-p1"], + patches = ["//patches:opusic-sys_materialize_install_headers.patch"], ) # The musl CLI's allocator needs declared native tools and consistent header flags. diff --git a/patches/BUILD.bazel b/patches/BUILD.bazel index 17410e8d402d..e60226a20e80 100644 --- a/patches/BUILD.bazel +++ b/patches/BUILD.bazel @@ -1,4 +1,5 @@ exports_files([ + "opusic-sys_materialize_install_headers.patch", "abseil_windows_gnullvm_thread_identity.patch", "bzip2_windows_stack_args.patch", "llvm_rusty_v8_custom_libcxx.patch", diff --git a/patches/opusic-sys_materialize_install_headers.patch b/patches/opusic-sys_materialize_install_headers.patch new file mode 100644 index 000000000000..f808371207df --- /dev/null +++ b/patches/opusic-sys_materialize_install_headers.patch @@ -0,0 +1,21 @@ +--- a/opus/CMakeLists.txt ++++ b/opus/CMakeLists.txt +@@ -278,6 +278,18 @@ + if(OPUS_CUSTOM_MODES) + list(APPEND Opus_PUBLIC_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/include/opus_custom.h) + endif() ++ ++# Bazel source inputs may be symlinks into an action's input tree. CMake's ++# install(FILES) preserves those links, which would escape the output tree. ++# Stage regular files for installation without changing header contents. ++set(Opus_INSTALL_HEADER) ++foreach(header IN LISTS Opus_PUBLIC_HEADER) ++ get_filename_component(header_name "${header}" NAME) ++ set(install_header "${CMAKE_CURRENT_BINARY_DIR}/install-headers/${header_name}") ++ configure_file("${header}" "${install_header}" COPYONLY) ++ list(APPEND Opus_INSTALL_HEADER "${install_header}") ++endforeach() ++set(Opus_PUBLIC_HEADER ${Opus_INSTALL_HEADER}) + + if(MSVC) + if(OPUS_STATIC_RUNTIME) From f560071053dd4c9efdc0325d4aa7514d65d66316 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 10:51:20 -0400 Subject: [PATCH 28/41] build: keep Opus patch context whitespace clean --- patches/opusic-sys_materialize_install_headers.patch | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/patches/opusic-sys_materialize_install_headers.patch b/patches/opusic-sys_materialize_install_headers.patch index f808371207df..1f2bef30b4b4 100644 --- a/patches/opusic-sys_materialize_install_headers.patch +++ b/patches/opusic-sys_materialize_install_headers.patch @@ -1,9 +1,6 @@ --- a/opus/CMakeLists.txt +++ b/opus/CMakeLists.txt -@@ -278,6 +278,18 @@ - if(OPUS_CUSTOM_MODES) - list(APPEND Opus_PUBLIC_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/include/opus_custom.h) - endif() +@@ -96,2 +96,14 @@ + +# Bazel source inputs may be symlinks into an action's input tree. CMake's +# install(FILES) preserves those links, which would escape the output tree. @@ -16,6 +13,5 @@ + list(APPEND Opus_INSTALL_HEADER "${install_header}") +endforeach() +set(Opus_PUBLIC_HEADER ${Opus_INSTALL_HEADER}) - if(MSVC) if(OPUS_STATIC_RUNTIME) From 3b1801b14208f4a7f77f54b6bd8ac575f82ceb29 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 10:51:53 -0400 Subject: [PATCH 29/41] build: align Opus patch anchor with pinned source --- patches/opusic-sys_materialize_install_headers.patch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/patches/opusic-sys_materialize_install_headers.patch b/patches/opusic-sys_materialize_install_headers.patch index 1f2bef30b4b4..f872223a5792 100644 --- a/patches/opusic-sys_materialize_install_headers.patch +++ b/patches/opusic-sys_materialize_install_headers.patch @@ -1,7 +1,6 @@ --- a/opus/CMakeLists.txt +++ b/opus/CMakeLists.txt -@@ -96,2 +96,14 @@ -+ +@@ -282,2 +282,14 @@ +# Bazel source inputs may be symlinks into an action's input tree. CMake's +# install(FILES) preserves those links, which would escape the output tree. +# Stage regular files for installation without changing header contents. @@ -13,5 +12,6 @@ + list(APPEND Opus_INSTALL_HEADER "${install_header}") +endforeach() +set(Opus_PUBLIC_HEADER ${Opus_INSTALL_HEADER}) ++ if(MSVC) if(OPUS_STATIC_RUNTIME) From e4ae9648e2eec208fc0269b43dfd1d80fdfab93c Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 11:01:56 -0400 Subject: [PATCH 30/41] Return the app-server time provider directly (Refs #18) --- codex-rs/app-server/src/message_processor.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index e1e5a7a9482e..49cce5a438ae 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -379,11 +379,10 @@ impl MessageProcessor { outgoing.clone(), thread_state_manager.clone(), )), - Some({ - let time_provider = - app_server_time_provider(outgoing.clone(), thread_state_manager.clone()); - time_provider - }), + Some(app_server_time_provider( + outgoing.clone(), + thread_state_manager.clone(), + )), ); match code_mode_session_provider { Some(provider) => manager.with_code_mode_session_provider(provider), From 82e148ece919d59717fca501bb4d1d78b52a0f10 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 11:53:20 -0400 Subject: [PATCH 31/41] ci: partition keyless Linux capacity across package shards --- .../linux_bazel_package_output_bytes.tsv | 176 +++++++ .github/scripts/linux_bazel_package_shards.py | 292 +++++++++++ .../test_ci_checkout_and_lint_policy.py | 2 + .../test_linux_bazel_package_shards.py | 476 ++++++++++++++++++ .github/workflows/bazel.yml | 181 +++++-- 5 files changed, 1086 insertions(+), 41 deletions(-) create mode 100644 .github/scripts/linux_bazel_package_output_bytes.tsv create mode 100644 .github/scripts/linux_bazel_package_shards.py create mode 100644 .github/scripts/test_linux_bazel_package_shards.py diff --git a/.github/scripts/linux_bazel_package_output_bytes.tsv b/.github/scripts/linux_bazel_package_output_bytes.tsv new file mode 100644 index 000000000000..95783d111c8a --- /dev/null +++ b/.github/scripts/linux_bazel_package_output_bytes.tsv @@ -0,0 +1,176 @@ +# Lower-bound logical producer-output weights; NOT physical bytes or dependency closures. +# Run40702301814 synthetic a914d089/treee6ad; max by package across GNU release artifact12827531079 and musl test artifact12826568075. +# Unknown packages default to1; all packages remain assigned. +bazel/build-info 58 +bazel/rules/testing/compat 218358280 +bazel/rules/testing/wine 27751591 +bazel/schema 4444236 +codex-rs/agent-graph-store 163485944 +codex-rs/agent-identity 153439550 +codex-rs/agent-message-board-client 402724785 +codex-rs/agent-roles 188181998 +codex-rs/analytics 362216202 +codex-rs/ansi-escape 17183188 +codex-rs/app-server 3423213406 +codex-rs/app-server-client 988035510 +codex-rs/app-server-daemon 411429182 +codex-rs/app-server-protocol 1043718167 +codex-rs/app-server-protocol-noop-macros 18832890 +codex-rs/app-server-test-client 1157548343 +codex-rs/app-server-transport 538195832 +codex-rs/app-server/tests/common 448200378 +codex-rs/apply-patch 867292652 +codex-rs/arg0 208075886 +codex-rs/async-utils 12713410 +codex-rs/attachment-store 12388569 +codex-rs/aws-auth 103049932 +codex-rs/backend-client 296576875 +codex-rs/build-info 21183108 +codex-rs/bwrap 1683557 +codex-rs/chatgpt 742440599 +codex-rs/cli 4581592 +codex-rs/cloud-client 66689551 +codex-rs/cloud-config 546991762 +codex-rs/cloud-tasks-client 78326286 +codex-rs/cloud-tasks-mock-client 75772032 +codex-rs/code-mode 206749086 +codex-rs/code-mode-host 1917030207 +codex-rs/code-mode-protocol 159726243 +codex-rs/code-mode-runtime 903221041 +codex-rs/codex-api 1086095468 +codex-rs/codex-backend-openapi-models 9649438 +codex-rs/codex-client 11146660 +codex-rs/codex-experimental-api-macros 30941950 +codex-rs/codex-home 229847363 +codex-rs/codex-mcp 582451626 +codex-rs/collaboration-mode-templates 7260628 +codex-rs/config 658341910 +codex-rs/config-schema 256622094 +codex-rs/connectors 236933120 +codex-rs/context-fragments 145188139 +codex-rs/core 2608252302 +codex-rs/core-api 208776292 +codex-rs/core-plugin-common 9493966 +codex-rs/core-plugins 601880950 +codex-rs/core/tests/common 485663481 +codex-rs/core/tests/remote_env_windows 818997992 +codex-rs/diagnostics 8228245 +codex-rs/direct-tool-timing 8460741 +codex-rs/exec 1177572356 +codex-rs/exec-server 7553285476 +codex-rs/exec-server-protocol 162766307 +codex-rs/exec-server/testing 710215794 +codex-rs/exec-server/tests/support 207068814 +codex-rs/execpolicy 326710199 +codex-rs/ext/agent 1027582600 +codex-rs/ext/agent-message-board 75546362 +codex-rs/ext/connectors 208807420 +codex-rs/ext/extension-api 951412890 +codex-rs/ext/git-attribution 316502903 +codex-rs/ext/goal 1138768841 +codex-rs/ext/guardian-reviewer 340717460 +codex-rs/ext/guardian-v2 856994334 +codex-rs/ext/history-notes 1833894388 +codex-rs/ext/image-generation 431909551 +codex-rs/ext/items 17028374 +codex-rs/ext/mcp 2187946370 +codex-rs/ext/memories 389889181 +codex-rs/ext/multi-env-copy-file 900102186 +codex-rs/ext/queue 1031538862 +codex-rs/ext/skills 1290562704 +codex-rs/ext/web-search 450047173 +codex-rs/external-agent-migration 579271288 +codex-rs/features 160072893 +codex-rs/feedback 278068071 +codex-rs/file-search 74683095 +codex-rs/file-system 150206868 +codex-rs/file-watcher 21107867 +codex-rs/git-utils 196407847 +codex-rs/guardian-context 314924344 +codex-rs/history 197501811 +codex-rs/hooks 519185904 +codex-rs/http-client 208344084 +codex-rs/install-context 11547024 +codex-rs/keyring-store 16917751 +codex-rs/linux-sandbox 830854639 +codex-rs/lmstudio 396180359 +codex-rs/login 1002542870 +codex-rs/memories/read 148254539 +codex-rs/memories/write 848842792 +codex-rs/mermaid 20028432 +codex-rs/message-board 478289111 +codex-rs/message-history 180373529 +codex-rs/model-provider 335882432 +codex-rs/model-provider-info 155228517 +codex-rs/models-manager 261942598 +codex-rs/mxc-sandbox 161279525 +codex-rs/network-proxy 637796513 +codex-rs/ollama 396393441 +codex-rs/otel 449600930 +codex-rs/otel-trace-websocket 7587440 +codex-rs/plugin 207645594 +codex-rs/process-hardening 7915102 +codex-rs/prompts 163267089 +codex-rs/protocol 399872035 +codex-rs/protocol/tests/support 75489859 +codex-rs/realtime-webrtc 60143150 +codex-rs/response-debug-context 148101260 +codex-rs/responses-api-proxy 150989127 +codex-rs/rmcp-client 7564049007 +codex-rs/rollout 370630251 +codex-rs/rollout-trace 197914372 +codex-rs/runtime-agent-instance 250150861 +codex-rs/sandboxing 221155659 +codex-rs/secrets 175016288 +codex-rs/shell-command 171478515 +codex-rs/skills 167902601 +codex-rs/state 229774434 +codex-rs/stdio-to-uds 46789846 +codex-rs/tcp-tunnel 58367290 +codex-rs/terminal-detection 8551655 +codex-rs/test-binary-support 75504744 +codex-rs/third-party/aws-config 99067801 +codex-rs/thread-manager-sample 1311808507 +codex-rs/thread-store 323984433 +codex-rs/tools 472962140 +codex-rs/tui 346174432 +codex-rs/uds 14041425 +codex-rs/user-verification 19893926 +codex-rs/utils/absolute-path 12191149 +codex-rs/utils/approval-presets 75524812 +codex-rs/utils/audio 148104326 +codex-rs/utils/cache 13927938 +codex-rs/utils/cargo-bin 8005042 +codex-rs/utils/cli 149051978 +codex-rs/utils/critical-path 8028333 +codex-rs/utils/elapsed 7297135 +codex-rs/utils/fuzzy-match 7576640 +codex-rs/utils/git-discovery 158883091 +codex-rs/utils/home-dir 10452925 +codex-rs/utils/image 40125424 +codex-rs/utils/json-to-toml 9435613 +codex-rs/utils/oss 366136461 +codex-rs/utils/output-truncation 145595538 +codex-rs/utils/path-uri 18390277 +codex-rs/utils/path-utils 12287169 +codex-rs/utils/plugins 261411330 +codex-rs/utils/process 7250766 +codex-rs/utils/pty 64803131 +codex-rs/utils/readiness 12269793 +codex-rs/utils/redacted-string 7322794 +codex-rs/utils/rustls-provider 36474691 +codex-rs/utils/sandbox-summary 146373903 +codex-rs/utils/sleep-inhibitor 8101175 +codex-rs/utils/stream-parser 10443308 +codex-rs/utils/string 10658862 +codex-rs/utils/template 8561028 +codex-rs/v8-poc 10332 +codex-rs/voice-host 267790476 +codex-rs/websocket-auth 20795084 +codex-rs/websocket-client 58920426 +codex-rs/windows-sandbox-rs 197521913 +codex-rs/windows-sandbox-rs/tests/support 4524 +codex-rs/windows-sandbox-service 198796022 +codex-rs/workload-identity 66037958 +codex-rs/worktree 474278891 +third_party/voice 103780378 diff --git a/.github/scripts/linux_bazel_package_shards.py b/.github/scripts/linux_bazel_package_shards.py new file mode 100644 index 000000000000..f35e013d8622 --- /dev/null +++ b/.github/scripts/linux_bazel_package_shards.py @@ -0,0 +1,292 @@ +"""Package-wildcard Linux shards; weights are estimates, never capacity proof.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +from hosted_storage import bounded_query + +FORK = "CompleteDotTech/codex-subagents-multi-provider" +COUNT = 4 +SCOPES = {"test": "//...", "release": "//codex-rs/..."} +WEIGHTS = Path(__file__).with_name("linux_bazel_package_output_bytes.tsv") +LABEL = re.compile(r"^//([A-Za-z0-9_./+\-]*):([A-Za-z0-9_./+@=,~\-]+)$") + + +def digest(value): + return hashlib.sha256(value).hexdigest() + + +def plan(env): + # Secret availability is evaluated by a step, never a job-if/matrix context. + availability = env.get("HAS_REMOTE_CREDENTIALS") + if availability not in ("true", "false"): + raise ValueError("explicit credential availability required") + split = ( + env.get("GITHUB_REPOSITORY") == FORK + and env.get("GITHUB_ACTIONS") == "true" + and env.get("RUNNER_ENVIRONMENT") == "github-hosted" + and env.get("RUNNER_OS") == "Linux" + and availability == "false" + ) + tests = [ + dict(os="macos-15-xlarge", target=target) + for target in ("aarch64-apple-darwin", "x86_64-apple-darwin") + ] + tests += [ + dict(os="ubuntu-24.04", target=target) + for target in ("x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl") + ] + release = [ + dict(os="ubuntu-24.04", target="x86_64-unknown-linux-gnu"), + dict(os="macos-15-xlarge", target="aarch64-apple-darwin"), + dict(os="windows-latest", target="x86_64-pc-windows-gnullvm"), + ] + + def rows(original): + result = [] + for row in original: + count = COUNT if split and row["os"] == "ubuntu-24.04" else 1 + for index in range(1, count + 1): + result.append( + dict( + row, + package_shard=index, + package_shard_count=count, + shard_suffix=f"-package-{index}-of-{count}" + if count > 1 + else "", + ) + ) + return {"include": result} + + # Keep the original upstream Windows runner group/label derivation. + repository = env.get("GITHUB_EVENT_REPOSITORY_NAME", "codex") + release[-1]["runs_on"] = { + "group": repository + "-runners", + "labels": repository + "-windows-x64", + } + return split, rows(tests), rows(release) + + +def labels(raw): + if len(raw) > 8 * 1024 * 1024: + raise ValueError("query output bound exceeded") + values = raw.decode("utf-8").splitlines() + if not values or len(values) > 250000 or len(values) != len(set(values)): + raise ValueError("empty, duplicated or oversized target universe") + for label in values: + match = LABEL.fullmatch(label) + if not match or any(p in (".", "..") for p in match[1].split("/")): + raise ValueError("invalid main-repository label") + if match[2] == "all": + raise ValueError("literal all target would shadow package wildcard") + return sorted(values) + + +def check_scope(mode, targets): + if mode == "release" and any( + not (x.startswith("//codex-rs/") or x.startswith("//codex-rs:")) + for x in targets + ): + raise ValueError("release package outside original scope") + + +def package_roots(targets): + return sorted({label.rsplit(":", 1)[0] + ":all" for label in targets}) + + +def read_weights(path=WEIGHTS): + result = {} + for line in path.read_text(encoding="utf-8").splitlines(): + if not line or line.startswith("#"): + continue + package, value = line.split("\t") + if package in result or not value.isdecimal() or int(value) < 1: + raise ValueError("invalid or duplicate package weight") + result["//" + package + ":all"] = int(value) + return result + + +def assign(roots, weights): + if len(roots) < COUNT or len(roots) != len(set(roots)): + raise ValueError("four nonempty disjoint package shards required") + shards = [[] for _ in range(COUNT)] + loads = [0] * COUNT + for root in sorted(roots, key=lambda x: (-weights.get(x, 1), x)): + index = min(range(COUNT), key=lambda i: (loads[i], i)) + shards[index].append(root) + loads[index] += weights.get(root, 1) + return [sorted(s) for s in shards], loads + + +def git_identity(): + # A query may update tracked lockfiles. Do not bind a changed checkout to + # HEAD's tree or accept package labels produced from untracked source files. + if subprocess.check_output( + ["git", "status", "--porcelain", "--untracked-files=normal"], timeout=5 + ).strip(): + raise ValueError("package manifest requires a clean source checkout") + + def rev(ref): + value = subprocess.check_output( + ["git", "rev-parse", ref], text=True, timeout=5 + ).strip() + if not re.fullmatch("[a-f0-9]{40}", value): + raise ValueError("invalid source identity") + return value + + return rev("HEAD"), rev("HEAD^{tree}") + + +def query(expression, env): + state, data = bounded_query( + [ + "bash", + str(Path(__file__).with_name("run-bazel-query-ci.sh")), + "--output=label", + "--", + expression, + ], + env, + timeout=240, + limit=8 * 1024 * 1024, + ) + if state != "ok": + raise ValueError("bounded package query unavailable: " + state) + return labels(data) + + +def create(mode, expected_head, env): + if not ( + env.get("GITHUB_ACTIONS") == "true" + and env.get("GITHUB_REPOSITORY") == FORK + and env.get("RUNNER_ENVIRONMENT") == "github-hosted" + and env.get("RUNNER_OS") == "Linux" + and not env.get("BUILDBUDDY_API_KEY") + ): + raise ValueError("package discovery is only for keyless hosted Linux fork") + head, tree = git_identity() + if head != expected_head: + raise ValueError("checkout does not match requested source") + original = query(SCOPES[mode], env) + check_scope(mode, original) + roots = package_roots(original) + expanded = query("set(" + " ".join(roots) + ")", env) + if original != expanded: + raise ValueError("package wildcard expansion changed original target universe") + if git_identity() != (head, tree): + raise ValueError("source changed during discovery") + assignments, loads = assign(roots, read_weights()) + return dict( + schema=1, + mode=mode, + head=head, + tree=tree, + scope=SCOPES[mode], + run_id=env["GITHUB_RUN_ID"], + attempt=env["GITHUB_RUN_ATTEMPT"], + original_labels=original, + expanded_labels=expanded, + labels_sha256=digest(("\n".join(original) + "\n").encode()), + weights_sha256=digest(WEIGHTS.read_bytes()), + assignments=assignments, + estimated_recorded_output_bytes=loads, + ) + + +def select(manifest, mode, expected_head, index, env): + if not 1 <= index <= COUNT: + raise ValueError("invalid package shard index") + head, tree = git_identity() + if ( + manifest.get("schema"), + manifest.get("mode"), + manifest.get("scope"), + manifest.get("head"), + manifest.get("tree"), + manifest.get("run_id"), + manifest.get("attempt"), + ) != ( + 1, + mode, + SCOPES[mode], + expected_head, + tree, + env["GITHUB_RUN_ID"], + env["GITHUB_RUN_ATTEMPT"], + ): + raise ValueError("manifest source, run or mode mismatch") + if head != expected_head or manifest.get("weights_sha256") != digest( + WEIGHTS.read_bytes() + ): + raise ValueError("consumer source or weights mismatch") + original = labels(("\n".join(manifest["original_labels"]) + "\n").encode()) + check_scope(mode, original) + expanded = labels(("\n".join(manifest["expanded_labels"]) + "\n").encode()) + if original != expanded or manifest["labels_sha256"] != digest( + ("\n".join(original) + "\n").encode() + ): + raise ValueError("manifest target conservation mismatch") + assignments, loads = assign(package_roots(original), read_weights()) + if ( + manifest["assignments"] != assignments + or manifest["estimated_recorded_output_bytes"] != loads + ): + raise ValueError("manifest assignment mismatch") + return assignments[index - 1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("plan", "create", "select")) + parser.add_argument("--mode", choices=SCOPES) + parser.add_argument("--expected-head") + parser.add_argument("--manifest", type=Path) + parser.add_argument("--shard", type=int) + args = parser.parse_args() + if args.operation == "plan": + split, tests, release = plan(os.environ) + with open( + os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8", newline="\n" + ) as out: + out.write("package-shards=" + str(split).lower() + "\n") + for key, value in (("test-matrix", tests), ("release-matrix", release)): + out.write(key + "=" + json.dumps(value, separators=(",", ":")) + "\n") + elif args.operation == "create": + if not args.mode or not args.expected_head or not args.manifest: + parser.error("create requires mode, source and manifest destination") + args.manifest.write_text( + json.dumps( + create(args.mode, args.expected_head, os.environ), sort_keys=True + ) + + "\n", + encoding="utf-8", + ) + else: + if ( + not args.mode + or not args.expected_head + or not args.manifest + or args.shard is None + ): + parser.error("select requires mode, source, manifest and shard") + if args.manifest.stat().st_size > 20 * 1024 * 1024: + raise ValueError("manifest file bound exceeded") + chosen = select( + json.loads(args.manifest.read_text(encoding="utf-8")), + args.mode, + args.expected_head, + args.shard, + os.environ, + ) + sys.stdout.buffer.write(("\n".join(chosen) + "\n").encode("utf-8")) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/test_ci_checkout_and_lint_policy.py b/.github/scripts/test_ci_checkout_and_lint_policy.py index 8d731e7a0a1c..ed8bb73d1810 100644 --- a/.github/scripts/test_ci_checkout_and_lint_policy.py +++ b/.github/scripts/test_ci_checkout_and_lint_policy.py @@ -13,6 +13,8 @@ class CheckoutAndLintPolicyTest(unittest.TestCase): def test_fork_cold_build_budgets_preserve_upstream_and_short_jobs(self): policies = { "bazel.yml": { + "plan-package-shards": ("5", None), + "linux-package-manifest": ("20", None), "test": ("30", 90), "test-windows-shard": ("30", 90), "test-windows-native-main": ("40", None), diff --git a/.github/scripts/test_linux_bazel_package_shards.py b/.github/scripts/test_linux_bazel_package_shards.py new file mode 100644 index 000000000000..3a9a33043a26 --- /dev/null +++ b/.github/scripts/test_linux_bazel_package_shards.py @@ -0,0 +1,476 @@ +"""No Bazel execution: routing, manifest conservation and real-shell argv probes.""" + +from collections import Counter +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import linux_bazel_package_shards as shards + +ROOT = Path(__file__).resolve().parents[2] +HEAD, TREE = "a" * 40, "b" * 40 +ENV = dict( + GITHUB_ACTIONS="true", + GITHUB_REPOSITORY=shards.FORK, + RUNNER_ENVIRONMENT="github-hosted", + RUNNER_OS="Linux", + HAS_REMOTE_CREDENTIALS="false", + GITHUB_RUN_ID="123", + GITHUB_RUN_ATTEMPT="1", + GITHUB_EVENT_REPOSITORY_NAME="codex-subagents-multi-provider", +) +TARGETS = sorted( + [ + "//codex-rs/a:binary", + "//codex-rs/a:manual-test", + "//codex-rs/a:suite", + "//codex-rs/b:unit-tests-bin", + "//codex-rs/c:library", + "//codex-rs/d:tool", + "//codex-rs/e:non-test", + "//codex-rs/f:custom", + ] +) + + +def job_block(name): + source = (ROOT / ".github/workflows/bazel.yml").read_text() + return re.search(r"^ " + name + r":\n(.*?)(?=^ [\w-]+:|\Z)", source, re.M | re.S)[ + 1 + ] + + +def step_script(job, name): + block = job_block(job).split(" - name: " + name + "\n", 1)[1] + run = block.split(" run: |\n", 1)[1].split("\n - ", 1)[0] + return "\n".join(line[10:] for line in run.splitlines()) + "\n" + + +def fixture_manifest(mode="test", identity=(HEAD, TREE)): + with ( + patch.object(shards, "git_identity", return_value=identity), + patch.object(shards, "query", side_effect=[TARGETS, TARGETS]), + ): + return shards.create(mode, identity[0], ENV) + + +class MatrixTests(unittest.TestCase): + def test_only_keyless_hosted_fork_linux_rows_expand_to_four_fresh_jobs(self): + enabled, tests, release = shards.plan(ENV) + self.assertTrue(enabled) + self.assertEqual( + Counter(x["target"] for x in tests["include"]), + { + "aarch64-apple-darwin": 1, + "x86_64-apple-darwin": 1, + "x86_64-unknown-linux-gnu": 4, + "x86_64-unknown-linux-musl": 4, + }, + ) + self.assertEqual(len(release["include"]), 6) + for matrix in (tests, release): + for target in { + x["target"] for x in matrix["include"] if x["os"] == "ubuntu-24.04" + }: + selected = [x for x in matrix["include"] if x["target"] == target] + self.assertEqual([x["package_shard"] for x in selected], [1, 2, 3, 4]) + self.assertEqual(len({x["shard_suffix"] for x in selected}), 4) + windows = release["include"][-1] + self.assertEqual( + windows["runs_on"], + { + "group": "codex-subagents-multi-provider-runners", + "labels": "codex-subagents-multi-provider-windows-x64", + }, + ) + + def test_keyed_upstream_other_repository_and_self_hosted_keep_original_rows(self): + for change in ( + {"HAS_REMOTE_CREDENTIALS": "true"}, + {"GITHUB_REPOSITORY": "openai/codex"}, + {"GITHUB_REPOSITORY": "other/repo"}, + {"RUNNER_ENVIRONMENT": "self-hosted"}, + {"RUNNER_OS": "Windows"}, + {"GITHUB_ACTIONS": "false"}, + ): + enabled, tests, release = shards.plan(dict(ENV, **change)) + self.assertFalse(enabled) + self.assertEqual(len(tests["include"]), 4) + self.assertEqual(len(release["include"]), 3) + for row in tests["include"] + release["include"]: + self.assertEqual( + (row["package_shard_count"], row["shard_suffix"]), (1, "") + ) + for value in (None, "", "unknown"): + with self.assertRaises(ValueError): + shards.plan(dict(ENV, HAS_REMOTE_CREDENTIALS=value)) + + +class ManifestTests(unittest.TestCase): + def test_dirty_or_untracked_source_cannot_be_bound_to_head_tree(self): + with patch.object( + shards.subprocess, "check_output", return_value=b" M MODULE.bazel.lock\n" + ): + with self.assertRaisesRegex(ValueError, "clean source"): + shards.git_identity() + with patch.object( + shards.subprocess, "check_output", return_value=b"?? new/BUILD.bazel\n" + ): + with self.assertRaisesRegex(ValueError, "clean source"): + shards.git_identity() + + def test_shared_manifest_proves_package_expansion_and_conserves_root_assignment( + self, + ): + manifest = fixture_manifest() + roots = shards.package_roots(TARGETS) + union = [] + with patch.object(shards, "git_identity", return_value=(HEAD, TREE)): + for index in range(1, 5): + union += shards.select(manifest, "test", HEAD, index, ENV) + self.assertEqual(sorted(union), roots) + self.assertEqual(len(union), len(set(union))) + self.assertEqual(manifest["original_labels"], TARGETS) + # Package wildcards retain manual rules, suites, binaries and libraries; + # they are not emitted as explicit test labels with altered semantics. + self.assertEqual(set(roots), {"//codex-rs/" + x + ":all" for x in "abcdef"}) + self.assertNotIn("//codex-rs/a:manual-test", union) + + def test_unequal_expansion_shadowing_duplicate_missing_and_invalid_labels_fail_closed( + self, + ): + for raw in ( + b"", + b"//x:a\n//x:a\n", + b"//x:all\n", + b"@external//x:a\n", + b"//../x:a\n", + b"//x:target;echo\n", + ): + with self.assertRaises(ValueError): + shards.labels(raw) + with ( + patch.object(shards, "git_identity", return_value=(HEAD, TREE)), + patch.object(shards, "query", side_effect=[TARGETS, TARGETS[:-1]]), + ): + with self.assertRaisesRegex(ValueError, "expansion"): + shards.create("test", HEAD, ENV) + with self.assertRaises(ValueError): + shards.assign(["//a:all"], {}) + with self.assertRaises(ValueError): + shards.assign(["//a:all"] * 4, {}) + + def test_source_run_mode_scope_assignment_and_digest_mismatches_are_rejected(self): + manifest = fixture_manifest() + changes = [ + {"head": "c" * 40}, + {"tree": "c" * 40}, + {"run_id": "other"}, + {"attempt": "2"}, + {"mode": "release"}, + {"scope": "//narrow/..."}, + {"weights_sha256": "bad"}, + {"labels_sha256": "bad"}, + {"assignments": [["//only:all"]] * 4}, + {"expanded_labels": TARGETS[:-1]}, + {"estimated_recorded_output_bytes": [0] * 4}, + ] + with patch.object(shards, "git_identity", return_value=(HEAD, TREE)): + for change in changes: + with self.subTest(change=change), self.assertRaises(ValueError): + shards.select(dict(manifest, **change), "test", HEAD, 1, ENV) + for index in (0, 5): + with self.assertRaises(ValueError): + shards.select(manifest, "test", HEAD, index, ENV) + with ( + patch.object( + shards, "git_identity", side_effect=[(HEAD, TREE), ("c" * 40, TREE)] + ), + patch.object(shards, "query", side_effect=[TARGETS, TARGETS]), + ): + with self.assertRaisesRegex(ValueError, "source changed"): + shards.create("test", HEAD, ENV) + with ( + patch.object(shards, "git_identity", return_value=(HEAD, TREE)), + patch.object(shards, "query", return_value=["//outside:binary"]), + ): + with self.assertRaisesRegex(ValueError, "scope"): + shards.create("release", HEAD, ENV) + + def test_query_is_time_output_bounded_and_cannot_succeed_with_partial_output(self): + with patch.object( + shards, "bounded_query", return_value=("ok", b"//a:x\n") + ) as query: + self.assertEqual(shards.query("//...", ENV), ["//a:x"]) + self.assertEqual( + query.call_args.kwargs, {"timeout": 240, "limit": 8 * 1024 * 1024} + ) + self.assertEqual( + query.call_args.args[0][-3:], ["--output=label", "--", "//..."] + ) + for status in ("timeout", "output-limit", "failed", "incomplete"): + with patch.object(shards, "bounded_query", return_value=(status, None)): + with self.assertRaises(ValueError): + shards.query("//...", ENV) + for change in ( + {"BUILDBUDDY_API_KEY": "synthetic"}, + {"RUNNER_OS": "Windows"}, + {"GITHUB_REPOSITORY": "openai/codex"}, + ): + with self.assertRaises(ValueError): + shards.create("test", HEAD, dict(ENV, **change)) + + def test_estimate_weights_do_not_omit_unknown_packages_or_change_tie_determinism( + self, + ): + roots = ["//p" + str(i) + ":all" for i in range(9)] + a, loads = shards.assign(roots, {roots[0]: 1000}) + self.assertEqual( + shards.assign(list(reversed(roots)), {roots[0]: 1000}), (a, loads) + ) + self.assertEqual(sorted(x for group in a for x in group), roots) + self.assertTrue(all(a)) + self.assertEqual(sum(loads), 1008) + + +class WorkflowTests(unittest.TestCase): + def test_contexts_required_manifest_and_existing_gate_arguments_are_preserved(self): + planner = job_block("plan-package-shards") + self.assertIn( + "HAS_REMOTE_CREDENTIALS: ${{ secrets.BUILDBUDDY_API_KEY != '' }}", planner + ) + self.assertNotIn("secrets.", planner.split(" steps:")[0]) + manifest = job_block("linux-package-manifest") + self.assertIn("if-no-files-found: error", manifest) + self.assertNotIn("continue-on-error", manifest) + for job in ("test", "verify-release-build"): + block = job_block(job) + self.assertIn("needs: [plan-package-shards, linux-package-manifest]", block) + self.assertIn("!cancelled()", block) + self.assertIn("needs.plan-package-shards.result == 'success'", block) + self.assertIn("needs.linux-package-manifest.result == 'success'", block) + self.assertIn( + "matrix: ${{ fromJSON(needs.plan-package-shards.outputs.", block + ) + self.assertNotIn("secrets.", block.split(" steps:")[0]) + self.assertIn( + "cache-scope: bazel-${{ github.job }}${{ matrix.shard_suffix }}", block + ) + self.assertIn( + "name: hosted-storage-${{ github.job }}-${{ matrix.target }}${{ matrix.shard_suffix }}", + block, + ) + self.assertIn('--expected-head "$GITHUB_SHA"', block) + self.assertIn('-- \\\n "${bazel_targets[@]}"', block) + test = step_script("test", "bazel test //...") + self.assertIn("--test_tag_filters=-argument-comment-lint", test) + self.assertIn("-//third_party/v8:all", test) + self.assertIn("-//codex-rs/v8-poc:v8-poc-unit-tests", test) + self.assertIn( + 'bazel_targets=("${package_roots[@]}" "${bazel_targets[@]:1}")', test + ) + release = step_script( + "verify-release-build", "bazel build verify-release-build targets" + ) + self.assertIn("extra_rustc_flag=-Cdebug-assertions=no", release) + self.assertIn("extra_exec_rustc_flag=-Cdebug-assertions=no", release) + self.assertIn("--compilation_mode=fastbuild", release) + self.assertIn( + 'bazel_targets=("${package_roots[@]}" "${bazel_targets[@]:1}")', release + ) + self.assertIn("Unexpected additional positive target pattern", release) + self.assertIn("//codex-rs/bwrap:bwrap", job_block("verify-release-build")) + + +class ShellArgumentTests(unittest.TestCase): + def test_unsharded_keyed_upstream_macos_and_windows_keep_original_target_arguments( + self, + ): + bash = ( + str( + Path(os.environ.get("ProgramFiles", "C:/Program Files")) + / "Git/bin/bash.exe" + ) + if os.name == "nt" + else shutil.which("bash") + ) + self.assertTrue(bash and Path(bash).exists()) + for mode, job, step in ( + ("test", "test", "bazel test //..."), + ( + "release", + "verify-release-build", + "bazel build verify-release-build targets", + ), + ): + for change in ( + {"BUILDBUDDY_API_KEY": "synthetic"}, + {"GITHUB_REPOSITORY": "openai/codex"}, + {"RUNNER_OS": "macOS"}, + {"RUNNER_OS": "Windows"}, + ): + with ( + self.subTest(mode=mode, change=change), + tempfile.TemporaryDirectory() as directory, + ): + temp = Path(directory) + record = temp / "argv.json" + stub = temp / "wrapper.py" + stub.write_text( + "import json,sys\nfrom pathlib import Path\nPath(" + + repr(str(record)) + + ").write_text(json.dumps(sys.argv[1:]))\n", + newline="\n", + ) + script = step_script(job, step).replace( + "./.github/scripts/run-bazel-ci.sh", + 'python3 "' + str(stub).replace("\\", "/") + '"', + ) + path = temp / "step.sh" + path.write_text("set -euo pipefail\n" + script, newline="\n") + env = { + k: v + for k, v in os.environ.items() + if k in ("PATH", "SYSTEMROOT", "SystemRoot", "TEMP", "TMP") + } + env.update( + ENV, + RUNNER_TEMP=str(temp).replace("\\", "/"), + GITHUB_SHA=HEAD, + LINUX_PACKAGE_SHARD="1", + LINUX_PACKAGE_SHARD_COUNT="1", + MSYS2_ARG_CONV_EXCL="*", + ) + env.update(change) + result = subprocess.run( + [bash, str(path)], + cwd=ROOT, + env=env, + text=True, + capture_output=True, + timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stderr) + argv = json.loads(record.read_text()) + second = [i for i, x in enumerate(argv) if x == "--"][1] + self.assertEqual( + argv[second + 1 :], + [ + "//...", + "-//third_party/v8:all", + "-//codex-rs/v8-poc:v8-poc-unit-tests", + ] + if mode == "test" + else [ + "//codex-rs/...", + "-//codex-rs/core/tests/remote_env_windows:smoke-test", + "-//codex-rs/v8-poc:all", + ], + ) + self.assertFalse((temp / "linux-package-manifests").exists()) + if mode == "release" and change.get("RUNNER_OS") == "Windows": + self.assertIn("--windows-cross-compile", argv) + + def test_real_shell_retains_package_wildcards_negatives_and_release_flags(self): + bash = ( + str( + Path(os.environ.get("ProgramFiles", "C:/Program Files")) + / "Git/bin/bash.exe" + ) + if os.name == "nt" + else shutil.which("bash") + ) + self.assertTrue( + bash and Path(bash).exists(), "Git Bash/bash required for this probe" + ) + identity = shards.git_identity() + for job, name, mode in ( + ("test", "bazel test //...", "test"), + ( + "verify-release-build", + "bazel build verify-release-build targets", + "release", + ), + ): + manifest = fixture_manifest(mode, identity) + with tempfile.TemporaryDirectory() as directory: + temp = Path(directory) + (temp / "linux-package-manifests").mkdir() + (temp / "linux-package-manifests" / (mode + ".json")).write_text( + json.dumps(manifest) + ) + record = temp / "record.json" + stub = temp / "wrapper.py" + stub.write_text( + "import json,sys\nfrom pathlib import Path\nPath(" + + repr(str(record)) + + ").write_text(json.dumps(sys.argv[1:]))\n", + newline="\n", + ) + script = step_script(job, name).replace( + "./.github/scripts/run-bazel-ci.sh", + 'python3 "' + str(stub).replace("\\", "/") + '"', + ) + path = temp / "step.sh" + path.write_text("set -euo pipefail\n" + script, newline="\n") + env = { + k: v + for k, v in os.environ.items() + if k in ("PATH", "SYSTEMROOT", "SystemRoot", "TEMP", "TMP") + } + env.update( + ENV, + RUNNER_TEMP=str(temp).replace("\\", "/"), + GITHUB_SHA=identity[0], + LINUX_PACKAGE_SHARD="1", + LINUX_PACKAGE_SHARD_COUNT="4", + MSYS2_ARG_CONV_EXCL="*", + ) + result = subprocess.run( + [bash, str(path)], + cwd=ROOT, + env=env, + text=True, + capture_output=True, + timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stderr) + argv = json.loads(record.read_text()) + # The wrapper's second separator starts targets; no target can + # become a flag or be expanded by the shell. + separators = [i for i, value in enumerate(argv) if value == "--"] + self.assertEqual(len(separators), 2) + payload = argv[separators[1] + 1 :] + self.assertEqual( + payload[: len(manifest["assignments"][0])], + manifest["assignments"][0], + ) + self.assertEqual( + payload[len(manifest["assignments"][0]) :], + ["-//third_party/v8:all", "-//codex-rs/v8-poc:v8-poc-unit-tests"] + if mode == "test" + else [ + "-//codex-rs/core/tests/remote_env_windows:smoke-test", + "-//codex-rs/v8-poc:all", + ], + ) + self.assertNotIn("//codex-rs/a:manual-test", payload) + if mode == "release": + self.assertIn( + "--@rules_rust//rust/settings:extra_rustc_flag=-Cdebug-assertions=no", + argv, + ) + self.assertIn( + "--@rules_rust//rust/settings:extra_exec_rustc_flag=-Cdebug-assertions=no", + argv, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 128286e223d0..0fcf67f5bed8 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -19,8 +19,67 @@ concurrency: cancel-in-progress: ${{ github.ref_name != 'main' }} jobs: - test: + plan-package-shards: if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic) }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + environment: + name: bazel + deployment: false + outputs: + package-shards: ${{ steps.plan.outputs.package-shards }} + test-matrix: ${{ steps.plan.outputs.test-matrix }} + release-matrix: ${{ steps.plan.outputs.release-matrix }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Plan unchanged routes or four fresh Linux package shards + id: plan + env: + HAS_REMOTE_CREDENTIALS: ${{ secrets.BUILDBUDDY_API_KEY != '' }} + GITHUB_EVENT_REPOSITORY_NAME: ${{ github.event.repository.name }} + run: python3 .github/scripts/linux_bazel_package_shards.py plan + + linux-package-manifest: + needs: plan-package-shards + if: needs.plan-package-shards.outputs.package-shards == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + environment: + name: bazel + deployment: false + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Prepare Bazel package discovery + uses: ./.github/actions/prepare-bazel-ci + with: + remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} + target: x86_64-unknown-linux-gnu + cache-scope: bazel-linux-package-manifest + - name: Prove source-bound package wildcard expansion + env: + BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/linux-package-manifests" + for mode in test release; do + python3 .github/scripts/linux_bazel_package_shards.py create \ + --mode "$mode" --expected-head "$GITHUB_SHA" \ + --manifest "$RUNNER_TEMP/linux-package-manifests/$mode.json" + done + - name: Publish required package manifests within this run + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: linux-package-manifests-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/linux-package-manifests/*.json + if-no-files-found: error + + test: + needs: [plan-package-shards, linux-package-manifest] + if: ${{ !cancelled() && !(github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic) && needs.plan-package-shards.result == 'success' && (needs.plan-package-shards.outputs.package-shards == 'false' || needs.linux-package-manifest.result == 'success') }} # PRs use the sharded Windows cross-compiled test jobs below. Post-merge # pushes to main also run the native Windows test job for broader Windows # signal without putting PR latency back on the critical path. When @@ -29,30 +88,11 @@ jobs: timeout-minutes: ${{ github.repository == 'openai/codex' && 30 || 90 }} strategy: fail-fast: false - matrix: - include: - # macOS - - os: macos-15-xlarge - target: aarch64-apple-darwin - - os: macos-15-xlarge - target: x86_64-apple-darwin - - # Linux - - os: ubuntu-24.04 - target: x86_64-unknown-linux-gnu - - os: ubuntu-24.04 - target: x86_64-unknown-linux-musl - # 2026-02-27 Bazel tests have been flaky on arm in CI. - # Disable until we can investigate and stabilize them. - # - os: ubuntu-24.04-arm - # target: aarch64-unknown-linux-musl - # - os: ubuntu-24.04-arm - # target: aarch64-unknown-linux-gnu - + matrix: ${{ fromJSON(needs.plan-package-shards.outputs.test-matrix) }} runs-on: ${{ github.repository == 'openai/codex' && matrix.os || fromJSON('{"macos-15-xlarge":"macos-15","windows-x64":"windows-2025","windows-latest":"windows-2025","windows-arm64":"windows-11-arm"}')[matrix.os] || matrix.os }} # Configure a human readable name for each job - name: Bazel test on ${{ matrix.os }} for ${{ matrix.target }} + name: Bazel test on ${{ matrix.os }} for ${{ matrix.target }}${{ matrix.shard_suffix }} environment: name: bazel deployment: false @@ -68,7 +108,7 @@ jobs: with: remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: ${{ matrix.target }} - cache-scope: bazel-${{ github.job }} + cache-scope: bazel-${{ github.job }}${{ matrix.shard_suffix }} - name: Check rusty_v8 MODULE.bazel checksums if: matrix.os == 'ubuntu-24.04' && matrix.target == 'x86_64-unknown-linux-gnu' @@ -92,9 +132,18 @@ jobs: releases="$(python3 .github/scripts/resolve_exec_server_releases.py)" echo "CODEX_COMPAT_RELEASES=$releases" >> "$GITHUB_ENV" + - name: Download required Linux package manifests + if: matrix.package_shard_count == 4 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: linux-package-manifests-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/linux-package-manifests + - name: bazel test //... env: BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + LINUX_PACKAGE_SHARD: ${{ matrix.package_shard }} + LINUX_PACKAGE_SHARD_COUNT: ${{ matrix.package_shard_count }} shell: bash run: | bazel_targets=( @@ -106,6 +155,31 @@ jobs: -//codex-rs/v8-poc:v8-poc-unit-tests ) + if [[ "$LINUX_PACKAGE_SHARD_COUNT" == 4 ]]; then + if [[ "$RUNNER_OS" != Linux || -n "${BUILDBUDDY_API_KEY:-}" ]]; then + echo "Package shard route changed after planning" >&2 + exit 1 + fi + selected_packages="$RUNNER_TEMP/linux-package-manifests/selected-test.txt" + python3 .github/scripts/linux_bazel_package_shards.py select \ + --mode test --expected-head "$GITHUB_SHA" --shard "$LINUX_PACKAGE_SHARD" \ + --manifest "$RUNNER_TEMP/linux-package-manifests/test.json" > "$selected_packages" + mapfile -t package_roots < "$selected_packages" + # Preserve the original negative patterns, not a second copied list. + # A future positive-scope change requires a new manifest contract. + if [[ "${bazel_targets[0]}" != "//..." ]]; then + echo "Original package scope changed" >&2 + exit 1 + fi + for target in "${bazel_targets[@]:1}"; do + if [[ "$target" != -//* ]]; then + echo "Unexpected additional positive target pattern" >&2 + exit 1 + fi + done + bazel_targets=("${package_roots[@]}" "${bazel_targets[@]:1}") + fi + bazel_wrapper_args=( --print-failed-action-summary --print-failed-test-logs @@ -128,7 +202,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: bazel-execution-logs-test-${{ matrix.target }} + name: bazel-execution-logs-test-${{ matrix.target }}${{ matrix.shard_suffix }} path: ${{ runner.temp }}/bazel-execution-logs if-no-files-found: ignore @@ -163,7 +237,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: hosted-storage-${{ github.job }}-${{ matrix.target }} + name: hosted-storage-${{ github.job }}-${{ matrix.target }}${{ matrix.shard_suffix }} path: ${{ runner.temp }}/hosted-storage/*.json if-no-files-found: ignore @@ -527,23 +601,14 @@ jobs: uses: ./.github/actions/check-clean-worktree verify-release-build: - if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic) }} + needs: [plan-package-shards, linux-package-manifest] + if: ${{ !cancelled() && !(github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic) && needs.plan-package-shards.result == 'success' && (needs.plan-package-shards.outputs.package-shards == 'false' || needs.linux-package-manifest.result == 'success') }} timeout-minutes: ${{ github.repository == 'openai/codex' && 30 || 90 }} strategy: fail-fast: false - matrix: - include: - - os: ubuntu-24.04 - target: x86_64-unknown-linux-gnu - - os: macos-15-xlarge - target: aarch64-apple-darwin - - os: windows-latest - target: x86_64-pc-windows-gnullvm - runs_on: - group: ${{ github.event.repository.name }}-runners - labels: ${{ github.event.repository.name }}-windows-x64 + matrix: ${{ fromJSON(needs.plan-package-shards.outputs.release-matrix) }} runs-on: ${{ github.repository == 'openai/codex' && (matrix.runs_on || matrix.os) || fromJSON('{"macos-15-xlarge":"macos-15","windows-x64":"windows-2025","windows-latest":"windows-2025","windows-arm64":"windows-11-arm"}')[matrix.os] || matrix.os }} - name: Verify release build on ${{ matrix.os }} for ${{ matrix.target }} + name: Verify release build on ${{ matrix.os }} for ${{ matrix.target }}${{ matrix.shard_suffix }} environment: name: bazel deployment: false @@ -564,7 +629,7 @@ jobs: with: remote-credentials: ${{ secrets.BUILDBUDDY_API_KEY != '' }} target: ${{ matrix.target }} - cache-scope: bazel-${{ github.job }} + cache-scope: bazel-${{ github.job }}${{ matrix.shard_suffix }} native-windows-voice: ${{ github.repository == 'CompleteDotTech/codex-subagents-multi-provider' && matrix.target == 'x86_64-pc-windows-gnullvm' && secrets.BUILDBUDDY_API_KEY == '' }} - name: Resolve app-server/exec-server compatibility releases @@ -575,9 +640,18 @@ jobs: releases="$(python3 .github/scripts/resolve_exec_server_releases.py)" echo "CODEX_COMPAT_RELEASES=$releases" >> "$GITHUB_ENV" + - name: Download required Linux package manifests + if: matrix.package_shard_count == 4 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: linux-package-manifests-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/linux-package-manifests + - name: bazel build verify-release-build targets env: BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + LINUX_PACKAGE_SHARD: ${{ matrix.package_shard }} + LINUX_PACKAGE_SHARD_COUNT: ${{ matrix.package_shard_count }} shell: bash run: | # This job exists to compile Rust code behind @@ -611,6 +685,31 @@ jobs: bazel_targets+=("${target}") done <<< "${bazel_target_lines}" + if [[ "$LINUX_PACKAGE_SHARD_COUNT" == 4 ]]; then + if [[ "$RUNNER_OS" != Linux || -n "${BUILDBUDDY_API_KEY:-}" ]]; then + echo "Package shard route changed after planning" >&2 + exit 1 + fi + selected_packages="$RUNNER_TEMP/linux-package-manifests/selected-release.txt" + python3 .github/scripts/linux_bazel_package_shards.py select \ + --mode release --expected-head "$GITHUB_SHA" --shard "$LINUX_PACKAGE_SHARD" \ + --manifest "$RUNNER_TEMP/linux-package-manifests/release.json" > "$selected_packages" + mapfile -t package_roots < "$selected_packages" + # Preserve the original negative patterns, not a second copied list. + # A future positive-scope change requires a new manifest contract. + if [[ "${bazel_targets[0]}" != "//codex-rs/..." ]]; then + echo "Original package scope changed" >&2 + exit 1 + fi + for target in "${bazel_targets[@]:1}"; do + if [[ "$target" != -//* ]]; then + echo "Unexpected additional positive target pattern" >&2 + exit 1 + fi + done + bazel_targets=("${package_roots[@]}" "${bazel_targets[@]:1}") + fi + ./.github/scripts/run-bazel-ci.sh \ "${bazel_wrapper_args[@]}" \ -- \ @@ -640,7 +739,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: bazel-execution-logs-verify-release-build-${{ matrix.target }} + name: bazel-execution-logs-verify-release-build-${{ matrix.target }}${{ matrix.shard_suffix }} path: ${{ runner.temp }}/bazel-execution-logs if-no-files-found: ignore @@ -675,7 +774,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: hosted-storage-${{ github.job }}-${{ matrix.target }} + name: hosted-storage-${{ github.job }}-${{ matrix.target }}${{ matrix.shard_suffix }} path: ${{ runner.temp }}/hosted-storage/*.json if-no-files-found: ignore From 2195a833733a34e4f51c4e2be5c95cd2af007bb0 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 10:09:02 -0400 Subject: [PATCH 32/41] test: isolate packaged bubblewrap controls from Bazel helper (cherry picked from commit 4e0c58caad3be5c773d6cce521669c335e7caf2d) --- codex-rs/linux-sandbox/tests/suite/bwrap_path.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codex-rs/linux-sandbox/tests/suite/bwrap_path.rs b/codex-rs/linux-sandbox/tests/suite/bwrap_path.rs index 2b95cbd3fc22..477fa4354d31 100644 --- a/codex-rs/linux-sandbox/tests/suite/bwrap_path.rs +++ b/codex-rs/linux-sandbox/tests/suite/bwrap_path.rs @@ -75,6 +75,8 @@ fn denied_dev_fd_starts_with_standard_devices_and_stdio_intact() { "test -L /dev/fd && test -d /dev/fd || exit 39" }; let mut command = Command::new(&sandbox); + // These package controls must not discover Bazel's separate declared helper. + command.env_remove("CARGO_BIN_EXE_bwrap"); if denied { command.env("PATH", &old_system_path); } @@ -105,6 +107,7 @@ fn denied_dev_fd_starts_with_standard_devices_and_stdio_intact() { } std::fs::remove_file(resources.join("bwrap")).unwrap(); let absent = Command::new(&sandbox) + .env_remove("CARGO_BIN_EXE_bwrap") .args([ "--sandbox-policy-cwd", "/", From 4b19293d14371c69e8eb695e9bd57c877bdaf4bf Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 27 Oct 2026 23:29:51 -0400 Subject: [PATCH 33/41] test(rollout): contextualize alternate group materialization failures (cherry picked from commit b192824107720a7d6d2531f7921c2d2ae98f1ce4) --- codex-rs/rollout/src/compression_tests.rs | 214 ++++++++++++++++++++-- 1 file changed, 195 insertions(+), 19 deletions(-) diff --git a/codex-rs/rollout/src/compression_tests.rs b/codex-rs/rollout/src/compression_tests.rs index 1005ba8616e9..8db775f80dea 100644 --- a/codex-rs/rollout/src/compression_tests.rs +++ b/codex-rs/rollout/src/compression_tests.rs @@ -742,39 +742,215 @@ async fn worker_preserves_inherited_acl_through_append() -> anyhow::Result<()> { Ok(()) } +#[cfg(unix)] +struct AlternateGroupDiagnostics { + remaining: usize, +} + +#[cfg(unix)] +impl AlternateGroupDiagnostics { + fn emit(&mut self, message: &str) { + use std::io::Write; + let line = format!("ROLLOUT_GID_DIAG {message}\n"); + let bytes = line.as_bytes(); + let count = bytes.len().min(self.remaining); + self.remaining -= count; + // Diagnostic output must never replace the original test result. + let _ = std::io::stderr().lock().write_all(&bytes[..count]); + } + + fn stage>( + &mut self, + stage: &'static str, + result: Result, + ) -> anyhow::Result { + match result { + Ok(value) => { + self.emit(&format!("stage={stage} completed=1")); + Ok(value) + } + Err(error) => { + let error = error.into(); + if let Some(io) = error.downcast_ref::() { + self.emit(&format!( + "stage={stage} completed=0 errno={:?}", + io.raw_os_error() + )); + } else { + self.emit(&format!("stage={stage} completed=0")); + } + Err(error.context(stage)) + } + } + } + + fn process(&mut self) { + use std::io::Read; + // Fixed files, bounded reads, selected numeric fields only; no environment, + // account names, arbitrary paths, or subprocesses are collected here. + for (role, path, limit) in [ + ("uid_map", "/proc/self/uid_map", 16384), + ("gid_map", "/proc/self/gid_map", 16384), + ("status", "/proc/self/status", 16384), + ("setgroups", "/proc/self/setgroups", 64), + ] { + let result = (|| -> std::io::Result> { + let mut bytes = Vec::new(); + fs::File::open(path)? + .take(limit + 1) + .read_to_end(&mut bytes)?; + Ok(bytes) + })(); + match result { + Ok(bytes) => { + self.emit(&format!( + "proc={role} truncated={}", + bytes.len() as u64 > limit + )); + let bytes = &bytes[..bytes.len().min(limit as usize)]; + if role == "setgroups" { + self.emit(&format!( + "proc=setgroups allow={} deny={}", + bytes == b"allow\n", + bytes == b"deny\n" + )); + continue; + } + for line in String::from_utf8_lossy(bytes).lines().take(64) { + let (field, values) = if role == "status" { + let Some((field, values)) = line.split_once(':') else { + continue; + }; + if !["Uid", "Gid", "Groups", "NoNewPrivs", "Seccomp", "CapEff"] + .contains(&field) + { + continue; + } + (field, values) + } else { + (role, line) + }; + let numeric: Vec<_> = values + .split_whitespace() + .take(32) + .filter(|value| value.bytes().all(|byte| byte.is_ascii_hexdigit())) + .collect(); + self.emit(&format!( + "proc={role} field={field} numeric={}", + numeric.join(" ") + )); + } + } + Err(error) => self.emit(&format!( + "proc={role} unavailable=1 errno={:?}", + error.raw_os_error() + )), + } + } + } + + fn file(&mut self, role: &'static str, path: &std::path::Path) { + use std::os::unix::fs::MetadataExt; + match fs::metadata(path) { + Ok(meta) => self.emit(&format!( + "file={role} uid={} gid={} dev={} ino={} mode={} mtime={} mtime_nsec={}", + meta.uid(), + meta.gid(), + meta.dev(), + meta.ino(), + meta.mode(), + meta.mtime(), + meta.mtime_nsec() + )), + Err(error) => self.emit(&format!( + "file={role} unavailable=1 errno={:?}", + error.raw_os_error() + )), + } + } +} + #[cfg(unix)] #[tokio::test] async fn worker_preserves_alternate_group_through_materialization() -> anyhow::Result<()> { use std::os::unix::fs::MetadataExt; use std::process::Command; - let home = TempDir::new()?; + let mut diagnostics = AlternateGroupDiagnostics { + remaining: 64 * 1024, + }; + diagnostics.process(); + let home = diagnostics.stage("create_home", TempDir::new())?; let uuid = Uuid::from_u128(62); - let thread_id = ThreadId::from_string(&uuid.to_string())?; + let thread_id = + diagnostics.stage("parse_thread_id", ThreadId::from_string(&uuid.to_string()))?; let path = archived_rollout_path(home.path(), "2025-01-03T12-00-00", uuid); - write_rollout(&path, thread_id, "different group")?; - set_old_mtime(&path)?; - let original_group = fs::metadata(&path)?.gid(); - let groups = Command::new("id").arg("-G").output()?; + diagnostics.stage( + "write_fixture", + write_rollout(&path, thread_id, "different group"), + )?; + diagnostics.stage("age_fixture", set_old_mtime(&path))?; + let original_group = diagnostics + .stage("read_original_metadata", fs::metadata(&path))? + .gid(); + let groups = diagnostics.stage("list_groups", Command::new("id").arg("-G").output())?; assert!(groups.status.success()); - let Some(group) = std::str::from_utf8(&groups.stdout)? - .split_whitespace() - .map(str::parse::) - .collect::, _>>()? - .into_iter() - .find(|group| *group != original_group) - else { + let groups_text = + diagnostics.stage("parse_groups_utf8", std::str::from_utf8(&groups.stdout))?; + let groups = diagnostics.stage( + "parse_groups", + groups_text + .split_whitespace() + .map(str::parse::) + .collect::, _>>(), + )?; + let Some(group) = groups.into_iter().find(|group| *group != original_group) else { eprintln!("skipping ownership regression: no alternate supplementary group"); return Ok(()); }; - std::os::unix::fs::chown(&path, /*uid*/ None, Some(group))?; - assert_eq!(fs::metadata(&path)?.gid(), group); + diagnostics.emit(&format!( + "selected_gid={group} original_gid={original_group}" + )); + diagnostics.file("before_chown", &path); + diagnostics.stage( + "set_alternate_group", + std::os::unix::fs::chown(&path, /*uid*/ None, Some(group)), + )?; + diagnostics.file("after_chown", &path); + assert_eq!( + diagnostics + .stage("verify_alternate_metadata", fs::metadata(&path))? + .gid(), + group + ); - worker::run(home.path().to_path_buf(), Startup).await?; + diagnostics.emit("stage=run_worker begin=1"); + diagnostics.stage( + "run_worker", + worker::run(home.path().to_path_buf(), Startup).await, + )?; assert!(!path.exists()); - assert_eq!(fs::metadata(compressed_rollout_path(&path))?.gid(), group); - materialize_rollout_for_append(&path, /*writer_lock*/ None).await?; - assert_eq!(fs::metadata(&path)?.gid(), group); + assert_eq!( + diagnostics + .stage( + "read_compressed_metadata", + fs::metadata(compressed_rollout_path(&path)) + )? + .gid(), + group + ); + diagnostics.file("before_materialize", &compressed_rollout_path(&path)); + diagnostics.emit("stage=materialize_append begin=1"); + diagnostics.stage( + "materialize_append", + materialize_rollout_for_append(&path, /*writer_lock*/ None).await, + )?; + assert_eq!( + diagnostics + .stage("verify_restored_metadata", fs::metadata(&path))? + .gid(), + group + ); Ok(()) } From 32870067ff70f3b94e02645ea47e2ee8f7861102 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 12:04:56 -0400 Subject: [PATCH 34/41] Require mapped host group coverage for rollout materialization (Refs #18) (cherry picked from commit 6396982b9c7090e7d830d7acc422283e7c2c36aa) --- codex-rs/rollout/BUILD.bazel | 24 +++++- codex-rs/rollout/src/compression_tests.rs | 96 ++++++++++++++++++++++- 2 files changed, 117 insertions(+), 3 deletions(-) diff --git a/codex-rs/rollout/BUILD.bazel b/codex-rs/rollout/BUILD.bazel index a91a3dd50731..34984ef5f415 100644 --- a/codex-rs/rollout/BUILD.bazel +++ b/codex-rs/rollout/BUILD.bazel @@ -1,6 +1,28 @@ -load("//:defs.bzl", "codex_rust_crate") +load("//:defs.bzl", "codex_rust_crate", "workspace_root_test") codex_rust_crate( name = "rollout", crate_name = "codex_rollout", ) + +# Linux sandbox user namespaces can expose unmapped supplementary groups as +# overflowgid. Keep the ordinary unit suite sandboxed, and require this exact +# ownership regression on a host where changing to a real member group works. +workspace_root_test( + name = "rollout-alternate-group-host-test", + size = "small", + args = [ + "compression::tests::worker_preserves_alternate_group_through_materialization", + "--exact", + "--test-threads=1", + ], + env = {"CODEX_ROLLOUT_REQUIRE_ALTERNATE_GROUP": "1"}, + tags = [ + "external", + "no-remote", + "no-sandbox", + ], + target_compatible_with = ["@platforms//os:linux"], + test_bin = ":rollout-unit-tests-bin", + workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker", +) diff --git a/codex-rs/rollout/src/compression_tests.rs b/codex-rs/rollout/src/compression_tests.rs index 8db775f80dea..57963e1f0bba 100644 --- a/codex-rs/rollout/src/compression_tests.rs +++ b/codex-rs/rollout/src/compression_tests.rs @@ -870,6 +870,68 @@ impl AlternateGroupDiagnostics { } } +#[cfg(target_os = "linux")] +fn mapped_alternate_group( + groups: &[u32], + original_group: u32, + gid_map: &str, +) -> anyhow::Result> { + // The first column is the namespace-visible ID range. In particular, an + // overflowgid reported by `id -G` is not necessarily a mapped member group. + let mut ranges = Vec::new(); + for line in gid_map.lines() { + let values = line + .split_whitespace() + .map(str::parse::) + .collect::, _>>()?; + anyhow::ensure!(values.len() == 3, "invalid gid_map column count"); + let end = values[0].checked_add(values[2]); + let parent_end = values[1].checked_add(values[2]); + anyhow::ensure!( + values[2] > 0 + && end.is_some_and(|end| end <= u64::from(u32::MAX)) + && parent_end.is_some_and(|end| end <= u64::from(u32::MAX)), + "invalid gid_map range" + ); + ranges.push((values[0], end.expect("validated range"))); + } + Ok(groups.iter().copied().find(|group| { + *group != original_group + && ranges + .iter() + .any(|(start, end)| *start <= u64::from(*group) && u64::from(*group) < *end) + })) +} + +#[cfg(target_os = "linux")] +#[test] +fn alternate_group_selection_requires_a_namespace_mapping() -> anyhow::Result<()> { + assert_eq!( + mapped_alternate_group(&[65534, 1000, 65534], 1000, "1000 1000 1\n")?, + None, + "unmapped supplementary groups must not masquerade as overflowgid membership" + ); + assert_eq!( + mapped_alternate_group(&[4, 1000], 1000, "0 0 4294967295\n")?, + Some(4) + ); + assert_eq!( + mapped_alternate_group(&[65534, 1000], 1000, "1000 1000 1\n65534 65534 1\n")?, + Some(65534), + "a genuinely mapped 65534 is valid; do not blacklist the numeric overflow default" + ); + assert_eq!(mapped_alternate_group(&[4, 1000], 1000, "")?, None); + for invalid in [ + "1000 1000", + "1 1 0", + "4294967295 1 1", + "1 18446744073709551615 1", + ] { + assert!(mapped_alternate_group(&[4, 1000], 1000, invalid).is_err()); + } + Ok(()) +} + #[cfg(unix)] #[tokio::test] async fn worker_preserves_alternate_group_through_materialization() -> anyhow::Result<()> { @@ -904,8 +966,37 @@ async fn worker_preserves_alternate_group_through_materialization() -> anyhow::R .map(str::parse::) .collect::, _>>(), )?; - let Some(group) = groups.into_iter().find(|group| *group != original_group) else { - eprintln!("skipping ownership regression: no alternate supplementary group"); + #[cfg(target_os = "linux")] + let group = { + let gid_map = + diagnostics.stage("read_gid_map", fs::read_to_string("/proc/self/gid_map"))?; + diagnostics.stage( + "select_mapped_group", + mapped_alternate_group(&groups, original_group, &gid_map), + )? + }; + #[cfg(not(target_os = "linux"))] + let group = groups.into_iter().find(|group| *group != original_group); + let required = match std::env::var_os("CODEX_ROLLOUT_REQUIRE_ALTERNATE_GROUP") { + None => false, + Some(value) => { + anyhow::ensure!( + value == "1", + "invalid required alternate-group test setting" + ); + true + } + }; + let Some(group) = group else { + anyhow::ensure!( + !required, + "required ownership regression has no mapped alternate supplementary group" + ); + diagnostics.emit("coverage=not_exercised reason=no_mapped_alternate_group"); + eprintln!( + "ownership regression not exercised: no mapped alternate supplementary group; \ + the required Linux host companion must exercise this case" + ); return Ok(()); }; diagnostics.emit(&format!( @@ -951,6 +1042,7 @@ async fn worker_preserves_alternate_group_through_materialization() -> anyhow::R .gid(), group ); + diagnostics.emit("coverage=exercised stages=chown_worker_materialize"); Ok(()) } From 91fedf0df8bb3b5c9914d116f5a5cba60dfa8ecb Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 12:46:23 -0400 Subject: [PATCH 35/41] Update Fence to supported hosted resolver release (Refs #18) --- .github/workflows/blob-size-policy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/blob-size-policy.yml b/.github/workflows/blob-size-policy.yml index f80fb0e4b30f..8424ef3e634f 100644 --- a/.github/workflows/blob-size-policy.yml +++ b/.github/workflows/blob-size-policy.yml @@ -10,7 +10,7 @@ jobs: permissions: contents: read steps: - - uses: openai/fence@e6b39c80c51cb2b2a39448ddeff74b3b886c0a63 # pin@v0.10.0 + - uses: openai/fence@f9751ca5d767dc0199e88733c2b286bd4800a008 # pin@v0.12.5 with: mode: audit - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 From 9d2a93bde3b6c120fcbd87c106143f307f53faf4 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 12:47:27 -0400 Subject: [PATCH 36/41] Include package planning jobs in diagnostic isolation checks (Refs #18) --- .../test_voice_runtime_diagnostic_workflow.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/scripts/test_voice_runtime_diagnostic_workflow.py b/.github/scripts/test_voice_runtime_diagnostic_workflow.py index f40d7bfda3c0..8f017a55f704 100644 --- a/.github/scripts/test_voice_runtime_diagnostic_workflow.py +++ b/.github/scripts/test_voice_runtime_diagnostic_workflow.py @@ -20,6 +20,8 @@ def setUp(self): def test_all_normal_jobs_guarded_and_original_predicates_retained(self): ordinary = { + "plan-package-shards", + "linux-package-manifest", "test", "test-windows-shard", "test-windows", @@ -30,6 +32,16 @@ def test_all_normal_jobs_guarded_and_original_predicates_retained(self): self.assertEqual(set(self.jobs), ordinary | {"voice-runtime-diagnostic"}) for name in ordinary: gate = re.search(r"^ if: (.+)$", self.jobs[name], re.M).group(1) + if name == "linux-package-manifest": + # The manifest inherits the planner's diagnostic exclusion: + # without a successful planner it cannot receive a true output. + self.assertIn(" needs: plan-package-shards\n", self.jobs[name]) + self.assertEqual( + gate, "needs.plan-package-shards.outputs.package-shards == 'true'" + ) + gate = re.search( + r"^ if: (.+)$", self.jobs["plan-package-shards"], re.M + ).group(1) self.assertIn( "!(github.event_name == 'workflow_dispatch' && inputs.voice_runtime_diagnostic)", gate, From c42912407ab3c3e6df877c2aec2f20a588f04bc4 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 13:01:42 -0400 Subject: [PATCH 37/41] Diagnose bounded PowerShell ownership fixture timeouts (Refs #18) --- .../test_windows_ci_resource_observation.py | 131 +++++++++++++++++- 1 file changed, 124 insertions(+), 7 deletions(-) diff --git a/.github/scripts/test_windows_ci_resource_observation.py b/.github/scripts/test_windows_ci_resource_observation.py index 0245f5c58169..5bbde2e60462 100644 --- a/.github/scripts/test_windows_ci_resource_observation.py +++ b/.github/scripts/test_windows_ci_resource_observation.py @@ -23,7 +23,98 @@ SPEC.loader.exec_module(m) +def ownership_fixture_timeout_summary(stderr): + """Retain only bounded fixed-stage/numeric observations, never raw errors.""" + stages = { + "entered", + "mocks-ready", + "collector-call", + "collector-return", + "query-memory", + "query-pagefiles", + "query-settings", + "query-volume", + "query-process", + "vhd-test", + "vhd-item", + } + if isinstance(stderr, bytes): + stderr = stderr[-8192:].decode("utf-8", errors="replace") + elif isinstance(stderr, str): + stderr = stderr[-8192:] + else: + stderr = "" + observed = [] + elapsed = None + version = None + for line in stderr.splitlines()[-64:]: + match = re.fullmatch( + r"OWNERSHIP_FIXTURE stage=([a-z-]+) elapsed_ms=([0-9]{1,5}) " + r"pwsh_major=([0-9]{1,3}) pwsh_minor=([0-9]{1,3})", + line, + ) + if not match or match[1] not in stages: + continue + milliseconds, major, minor = map(int, match.group(2, 3, 4)) + if milliseconds > 60000 or major > 100 or minor > 100: + continue + if not observed or observed[-1] != match[1]: + observed.append(match[1]) + elapsed, version = milliseconds, [major, minor] + return "ownership-fixture-timeout " + json.dumps( + {"stages": observed[-16:], "elapsed_ms": elapsed, "pwsh_version": version}, + separators=(",", ":"), + ) + + class ObservationTests(unittest.TestCase): + def test_ownership_timeout_summary_retains_only_known_stage_and_numbers(self): + summary = ownership_fixture_timeout_summary( + b"SECRET endpoint/path\n" + b"OWNERSHIP_FIXTURE stage=entered elapsed_ms=0 pwsh_major=7 pwsh_minor=4\n" + b"OWNERSHIP_FIXTURE stage=collector-call elapsed_ms=120 pwsh_major=7 pwsh_minor=4\n" + b"SECRET token\n" + ) + data = json.loads(summary.split(" ", 1)[1]) + self.assertEqual(data["stages"], ["entered", "collector-call"]) + self.assertEqual(data["elapsed_ms"], 120) + self.assertEqual(data["pwsh_version"], [7, 4]) + self.assertNotIn("SECRET", summary) + + def test_ownership_timeout_summary_missing_or_unrecognized_is_unknown(self): + for value in ( + None, + "SECRET", + b"SECRET", + "OWNERSHIP_FIXTURE stage=secret-path elapsed_ms=1 pwsh_major=7 pwsh_minor=4", + "OWNERSHIP_FIXTURE stage=entered elapsed_ms=99999 pwsh_major=7 pwsh_minor=4", + "OWNERSHIP_FIXTURE stage=entered elapsed_ms=1 pwsh_major=999 pwsh_minor=4", + ): + with self.subTest(value=type(value).__name__): + self.assertEqual( + json.loads( + ownership_fixture_timeout_summary(value).split(" ", 1)[1] + ), + {"stages": [], "elapsed_ms": None, "pwsh_version": None}, + ) + + def test_ownership_timeout_diagnostics_never_turn_timeout_into_success(self): + for stderr in ( + b"SECRET\nOWNERSHIP_FIXTURE stage=entered elapsed_ms=0 pwsh_major=7 pwsh_minor=4\n", + b"SECRET unrecognized failure", + ): + error = subprocess.TimeoutExpired(["SECRET"], 10, stderr=stderr) + output = io.StringIO() + with ( + patch.object(subprocess, "run", side_effect=error), + contextlib.redirect_stderr(output), + self.assertRaises(subprocess.TimeoutExpired) as raised, + ): + self.run_ownership_fixture() + self.assertIs(raised.exception, error) + self.assertIn("ownership-fixture-timeout ", output.getvalue()) + self.assertNotIn("SECRET", output.getvalue()) + def test_scope_requires_every_gate_and_actual_windows(self): env = dict( GITHUB_REPOSITORY="CompleteDotTech/codex-subagents-multi-provider", @@ -373,10 +464,27 @@ def run_ownership_fixture( ): script = Path(__file__).with_name("observe_windows_ci_resources.ps1").resolve() fixture = r""" +$global:ownershipFixtureClock = [Diagnostics.Stopwatch]::StartNew() +$global:ownershipFixtureStageCount = 0 +function global:Write-OwnershipFixtureStage { + param([string]$Stage) + if ($global:ownershipFixtureStageCount -ge 64 -and $Stage -ne 'collector-return') { return } + $global:ownershipFixtureStageCount++ + $elapsed = [Math]::Min(60000, $global:ownershipFixtureClock.ElapsedMilliseconds) + [Console]::Error.WriteLine("OWNERSHIP_FIXTURE stage=$Stage elapsed_ms=$elapsed pwsh_major=$($PSVersionTable.PSVersion.Major) pwsh_minor=$($PSVersionTable.PSVersion.Minor)") +} +Write-OwnershipFixtureStage 'entered' $global:fixtureQueries = 0 $global:fixtureCreated = [datetime]::new(2026, 1, 1, 0, 0, 0, [DateTimeKind]::Utc) function global:Get-CimInstance { param($ClassName, $Filter, $OperationTimeoutSec, $ErrorAction) + switch ($ClassName) { + 'Win32_PerfFormattedData_PerfOS_Memory' { Write-OwnershipFixtureStage 'query-memory' } + 'Win32_PageFileUsage' { Write-OwnershipFixtureStage 'query-pagefiles' } + 'Win32_PageFileSetting' { Write-OwnershipFixtureStage 'query-settings' } + 'Win32_LogicalDisk' { Write-OwnershipFixtureStage 'query-volume' } + 'Win32_Process' { Write-OwnershipFixtureStage 'query-process' } + } switch ($ClassName) { 'Win32_PerfFormattedData_PerfOS_Memory' { [pscustomobject]@{ AvailableBytes=$null; CommittedBytes=200; CommitLimit=400; PercentCommittedBytesInUse=50 } } 'Win32_PageFileUsage' { [pscustomobject]@{ Name='C:\pagefile.sys'; AllocatedBaseSize=$null; CurrentUsage=1; PeakUsage=2 } } @@ -406,11 +514,13 @@ def run_ownership_fixture( } function global:Test-Path { param($LiteralPath, $PathType) + Write-OwnershipFixtureStage 'vhd-test' if ($LiteralPath -ne $global:fixtureVhd) { throw 'unexpected VHD path' } return $true } function global:Get-Item { param($LiteralPath) + Write-OwnershipFixtureStage 'vhd-item' if ($LiteralPath -ne $global:fixtureVhd) { throw 'unexpected VHD path' } [pscustomobject]@{ Length=4096 } } @@ -418,7 +528,10 @@ def run_ownership_fixture( $global:fixtureVhd = Join-Path $env:RUNNER_TEMP 'codex-dev-drive.vhdx' $epoch = [datetime]::new(1601, 1, 1, 0, 0, 0, [DateTimeKind]::Utc).Ticks $micro = [long](($global:fixtureCreated.Ticks - $epoch) / 10D) + OFFSET +Write-OwnershipFixtureStage 'mocks-ready' +Write-OwnershipFixtureStage 'collector-call' & $args[0] -OwnedPid 101 -CreationMicroseconds $micro +Write-OwnershipFixtureStage 'collector-return' """ # Keep Windows C: semantics there; Linux PowerShell needs a native # filesystem root for Join-Path and System.IO.Path root interpretation. @@ -436,13 +549,17 @@ def run_ownership_fixture( with tempfile.TemporaryDirectory() as temp: runner = Path(temp) / "fixture.ps1" runner.write_text(fixture) - result = subprocess.run( - ["pwsh", "-NoProfile", "-File", str(runner), str(script)], - capture_output=True, - text=True, - timeout=10, - check=True, - ) + try: + result = subprocess.run( + ["pwsh", "-NoProfile", "-File", str(runner), str(script)], + capture_output=True, + text=True, + timeout=10, + check=True, + ) + except subprocess.TimeoutExpired as error: + print(ownership_fixture_timeout_summary(error.stderr), file=sys.stderr) + raise return json.loads(result.stdout) def test_powershell_exact_root_identity_and_numeric_vhd_fixture(self): From ce3d12901087b0eaaae50611c056e45a5a90fce3 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 09:14:59 -0400 Subject: [PATCH 38/41] Specify V8 archive type for offline cache extraction (Refs #62) (cherry picked from commit 1ba099ad1d9fb1ae584d27bc6adf2344ae90cfee) --- MODULE.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/MODULE.bazel b/MODULE.bazel index ca1de5dc75a8..35b046592d74 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -829,6 +829,7 @@ archive_override( "//patches:v8_array_sort_elements_kind.patch", ], strip_prefix = "v8-15.2.124.1", + type = "tar.gz", urls = ["https://github.com/v8/v8/archive/refs/tags/15.2.124.1.tar.gz"], ) From c59c0d972f78391970a9e58e323ecec393221bc6 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 13:27:31 -0400 Subject: [PATCH 39/41] Expose bounded first-entry evidence for PowerShell fixture timeout (Refs #18) (cherry picked from commit fdc4d2eb5382b1a661e69a8780f33fcccbd468f5) --- .../test_windows_ci_resource_observation.py | 118 ++++++++++++++++-- 1 file changed, 110 insertions(+), 8 deletions(-) diff --git a/.github/scripts/test_windows_ci_resource_observation.py b/.github/scripts/test_windows_ci_resource_observation.py index 5bbde2e60462..379b195762b5 100644 --- a/.github/scripts/test_windows_ci_resource_observation.py +++ b/.github/scripts/test_windows_ci_resource_observation.py @@ -23,7 +23,31 @@ SPEC.loader.exec_module(m) -def ownership_fixture_timeout_summary(stderr): +def ownership_capture_metadata(value): + """Count only bounded bytes; text counts use UTF-8 replacement encoding.""" + if isinstance(value, bytes): + size = len(value) + kind = "bytes" + elif isinstance(value, str): + # At most 8193 characters are encoded, even for an oversized capture. + size = len(value[:8193].encode("utf-8", errors="replace")) + kind = "text" + else: + return { + "present": value is not None, + "kind": "none" if value is None else "unsupported", + "byte_count": None, + "byte_count_clamped": False, + } + return { + "present": True, + "kind": kind, + "byte_count": min(size, 8192), + "byte_count_clamped": size > 8192, + } + + +def ownership_fixture_timeout_summary(stderr, stdout=None): """Retain only bounded fixed-stage/numeric observations, never raw errors.""" stages = { "entered", @@ -38,6 +62,10 @@ def ownership_fixture_timeout_summary(stderr): "vhd-test", "vhd-item", } + captures = { + "stdout_capture": ownership_capture_metadata(stdout), + "stderr_capture": ownership_capture_metadata(stderr), + } if isinstance(stderr, bytes): stderr = stderr[-8192:].decode("utf-8", errors="replace") elif isinstance(stderr, str): @@ -47,7 +75,8 @@ def ownership_fixture_timeout_summary(stderr): observed = [] elapsed = None version = None - for line in stderr.splitlines()[-64:]: + retained_lines = stderr.splitlines()[-64:] + for line in retained_lines: match = re.fullmatch( r"OWNERSHIP_FIXTURE stage=([a-z-]+) elapsed_ms=([0-9]{1,5}) " r"pwsh_major=([0-9]{1,3}) pwsh_minor=([0-9]{1,3})", @@ -62,7 +91,19 @@ def ownership_fixture_timeout_summary(stderr): observed.append(match[1]) elapsed, version = milliseconds, [major, minor] return "ownership-fixture-timeout " + json.dumps( - {"stages": observed[-16:], "elapsed_ms": elapsed, "pwsh_version": version}, + { + "stages": observed[-16:], + "elapsed_ms": elapsed, + "pwsh_version": version, + "literal_entry_seen_in_retained_stderr": "OWNERSHIP_FIXTURE_ENTRY" + in retained_lines, + "capture_byte_count_limit": 8192, + "text_byte_count_encoding": "utf8-replace", + "stderr_window_limit": 8192, + "stderr_window_units": "bytes-before-utf8-decode-or-text-characters", + "stderr_line_limit": 64, + **captures, + }, separators=(",", ":"), ) @@ -92,18 +133,75 @@ def test_ownership_timeout_summary_missing_or_unrecognized_is_unknown(self): ): with self.subTest(value=type(value).__name__): self.assertEqual( - json.loads( - ownership_fixture_timeout_summary(value).split(" ", 1)[1] - ), + { + key: result + for key, result in json.loads( + ownership_fixture_timeout_summary(value).split(" ", 1)[1] + ).items() + if key in {"stages", "elapsed_ms", "pwsh_version"} + }, {"stages": [], "elapsed_ms": None, "pwsh_version": None}, ) + def test_ownership_timeout_capture_metadata_distinguishes_absence_and_empty(self): + for value, present, kind, count in ( + (None, False, "none", None), + (b"", True, "bytes", 0), + ("", True, "text", 0), + (b"SECRET", True, "bytes", 6), + ("é", True, "text", 2), + (b"\xff", True, "bytes", 1), + (42, True, "unsupported", None), + ): + summary = ownership_fixture_timeout_summary(value, value) + data = json.loads(summary.split(" ", 1)[1]) + for key in ("stdout_capture", "stderr_capture"): + self.assertEqual( + data[key], + { + "present": present, + "kind": kind, + "byte_count": count, + "byte_count_clamped": False, + }, + ) + self.assertFalse(data["literal_entry_seen_in_retained_stderr"]) + self.assertEqual(data["stages"], []) + self.assertNotIn("SECRET", summary) + + def test_ownership_timeout_literal_and_capture_windows_are_bounded(self): + literal = "OWNERSHIP_FIXTURE_ENTRY" + for value in (literal, literal.encode()): + data = json.loads(ownership_fixture_timeout_summary(value).split(" ", 1)[1]) + self.assertTrue(data["literal_entry_seen_in_retained_stderr"]) + self.assertEqual(data["stages"], []) + self.assertIsNone(data["elapsed_ms"]) + for value in ( + "prefix" + literal, + literal + " suffix", + literal + "\n" + "\n".join(["SECRET"] * 65), + literal + "\n" + "é" * 9000, + literal.encode() + b"\n" + b"SECRET" * 2000, + ): + summary = ownership_fixture_timeout_summary(value, value) + data = json.loads(summary.split(" ", 1)[1]) + self.assertFalse(data["literal_entry_seen_in_retained_stderr"]) + self.assertEqual(data["stderr_window_limit"], 8192) + self.assertEqual(data["stderr_line_limit"], 64) + self.assertNotIn("SECRET", summary) + if len(value) > 8192: + for key in ("stdout_capture", "stderr_capture"): + self.assertEqual(data[key]["byte_count"], 8192) + self.assertTrue(data[key]["byte_count_clamped"]) + def test_ownership_timeout_diagnostics_never_turn_timeout_into_success(self): for stderr in ( b"SECRET\nOWNERSHIP_FIXTURE stage=entered elapsed_ms=0 pwsh_major=7 pwsh_minor=4\n", b"SECRET unrecognized failure", ): - error = subprocess.TimeoutExpired(["SECRET"], 10, stderr=stderr) + error = subprocess.TimeoutExpired( + ["SECRET"], 10, output=b"SECRET stdout", stderr=stderr + ) output = io.StringIO() with ( patch.object(subprocess, "run", side_effect=error), @@ -464,6 +562,7 @@ def run_ownership_fixture( ): script = Path(__file__).with_name("observe_windows_ci_resources.ps1").resolve() fixture = r""" +[Console]::Error.WriteLine('OWNERSHIP_FIXTURE_ENTRY') $global:ownershipFixtureClock = [Diagnostics.Stopwatch]::StartNew() $global:ownershipFixtureStageCount = 0 function global:Write-OwnershipFixtureStage { @@ -558,7 +657,10 @@ def run_ownership_fixture( check=True, ) except subprocess.TimeoutExpired as error: - print(ownership_fixture_timeout_summary(error.stderr), file=sys.stderr) + print( + ownership_fixture_timeout_summary(error.stderr, error.stdout), + file=sys.stderr, + ) raise return json.loads(result.stdout) From 817c84adfa339f5a7c7ac99aec3d3162f6967c40 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 13:16:41 -0400 Subject: [PATCH 40/41] Diagnose managed proxy helper visibility without widening access (Refs #81) (cherry picked from commit 2142db83bcf43087376bbdba0aa54ddaed82049d) --- .../tests/suite/managed_proxy.rs | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs b/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs index 49f47f342a70..9652c4fc340f 100644 --- a/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs +++ b/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs @@ -21,6 +21,8 @@ use codex_utils_absolute_path::AbsolutePathBuf; use pretty_assertions::assert_eq; use std::collections::HashMap; use std::io::Read; +use std::io::Seek; +use std::io::SeekFrom; use std::io::Write; use std::net::Ipv4Addr; use std::net::Shutdown; @@ -191,6 +193,114 @@ fn linux_sandbox_command( cmd } +// Observe only the declared test helper, never arbitrary environment paths. An +// interpreter path is used internally for fixed numeric facts, not printed. +fn helper_elf_interpreter(path: &Path) -> Result, &'static str> { + if !std::fs::metadata(path).map_err(|_| "metadata")?.is_file() { + return Err("not-regular"); + } + let mut file = std::fs::File::open(path).map_err(|_| "open")?; + let size = file.metadata().map_err(|_| "metadata")?.len(); + let mut header = [0_u8; 64]; + file.read_exact(&mut header).map_err(|_| "header")?; + if &header[..4] != b"\x7fELF" || header[4] != 2 || header[5] != 1 { + return Err("unsupported-format"); + } + let offset = u64::from_le_bytes(header[32..40].try_into().map_err(|_| "header")?); + let entry_size = u16::from_le_bytes([header[54], header[55]]); + let count = u16::from_le_bytes([header[56], header[57]]); + if entry_size != 56 + || count > 128 + || offset + .checked_add(u64::from(count) * u64::from(entry_size)) + .is_none_or(|end| end > size) + { + return Err("program-headers"); + } + for index in 0..count { + file.seek(SeekFrom::Start(offset + u64::from(index) * 56)) + .map_err(|_| "program-header")?; + let mut entry = [0_u8; 56]; + file.read_exact(&mut entry).map_err(|_| "program-header")?; + if u32::from_le_bytes(entry[..4].try_into().map_err(|_| "program-header")?) != 3 { + continue; + } + let start = u64::from_le_bytes(entry[8..16].try_into().map_err(|_| "interpreter")?); + let length = u64::from_le_bytes(entry[32..40].try_into().map_err(|_| "interpreter")?); + if !(2..=4096).contains(&length) || start.checked_add(length).is_none_or(|end| end > size) { + return Err("interpreter"); + } + file.seek(SeekFrom::Start(start)) + .map_err(|_| "interpreter")?; + let mut bytes = vec![0; length as usize]; + file.read_exact(&mut bytes).map_err(|_| "interpreter")?; + if bytes.pop() != Some(0) || bytes.contains(&0) { + return Err("interpreter"); + } + let value = String::from_utf8(bytes).map_err(|_| "interpreter")?; + let path = std::path::PathBuf::from(value); + if !path.is_absolute() { + return Err("interpreter"); + } + return Ok(Some(path)); + } + Ok(None) +} + +fn managed_proxy_path_facts(role: &str, path: &Path, declared_root: &Path) -> String { + let metadata = std::fs::metadata(path).ok(); + let canonical = path.canonicalize().ok(); + let canonical_root = declared_root.canonicalize().ok(); + format!( + "MANAGED_PROXY_PATH_DIAG role={role} observation=host-after-failure absolute={} regular={} symlink={} exec_mode={} identity={:?} canonical_equals_logical={} declared_root_contains_canonical={} canonical_root_contains_canonical={}\n", + path.is_absolute(), + metadata.as_ref().is_some_and(|info| info.is_file()), + std::fs::symlink_metadata(path) + .ok() + .is_some_and(|info| info.file_type().is_symlink()), + metadata + .as_ref() + .is_some_and(|info| info.mode() & 0o111 != 0), + metadata.as_ref().map(|info| (info.dev(), info.ino())), + canonical.as_ref().is_some_and(|value| value == path), + canonical + .as_ref() + .is_some_and(|value| value.starts_with(declared_root)), + canonical.as_ref().is_some_and(|value| canonical_root + .as_ref() + .is_some_and(|root| value.starts_with(root))), + ) +} + +fn emit_managed_proxy_path_diagnostics(helper: &Path, declared_root: &Path) { + let mut facts = managed_proxy_path_facts("declared-helper", helper, declared_root); + if let Ok(test_runner) = std::env::current_exe() { + // This is the integration test executable, NOT the sandbox's current_exe. + facts.push_str(&managed_proxy_path_facts( + "test-runner", + &test_runner, + declared_root, + )); + } + match helper_elf_interpreter(helper) { + Ok(Some(interpreter)) => facts.push_str(&managed_proxy_path_facts( + "helper-interpreter", + &interpreter, + declared_root, + )), + Ok(None) => facts.push_str("MANAGED_PROXY_PATH_DIAG interpreter=absent\n"), + Err(reason) => facts.push_str(&format!( + "MANAGED_PROXY_PATH_DIAG interpreter=unknown reason={reason}\n" + )), + } + // Direct bounded stderr survives libtest buffering. Diagnostics never replace + // the original failure, and do not probe or broaden the inner sandbox view. + let bytes = facts.as_bytes(); + let _ = std::io::stderr() + .lock() + .write_all(&bytes[..bytes.len().min(4096)]); +} + #[tokio::test] async fn sandboxed_commands_have_a_seccomp_filtered_namespace_reaper() { if should_skip_bwrap_tests().await { @@ -830,6 +940,12 @@ async fn managed_proxy_mode_routes_through_bridge_and_blocks_direct_egress() { ) .await; + if !routed_output.status.success() { + emit_managed_proxy_path_diagnostics( + Path::new(env!("CARGO_BIN_EXE_codex-linux-sandbox")), + sandbox_helper_dir, + ); + } assert_eq!( routed_output.status.success(), true, From d7bddc80065a74e4636aa30930250e08d4fa2477 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 28 Oct 2026 13:37:22 -0400 Subject: [PATCH 41/41] Distinguish unavailable proxy path observations (Refs #81) (cherry picked from commit b8a8e577a0ee5601f463210ca9cb40be359a9b64) --- .../tests/suite/managed_proxy.rs | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs b/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs index 9652c4fc340f..c34806605f79 100644 --- a/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs +++ b/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs @@ -251,24 +251,24 @@ fn managed_proxy_path_facts(role: &str, path: &Path, declared_root: &Path) -> St let metadata = std::fs::metadata(path).ok(); let canonical = path.canonicalize().ok(); let canonical_root = declared_root.canonicalize().ok(); + // Optional facts distinguish unavailable observations from observed false values. format!( - "MANAGED_PROXY_PATH_DIAG role={role} observation=host-after-failure absolute={} regular={} symlink={} exec_mode={} identity={:?} canonical_equals_logical={} declared_root_contains_canonical={} canonical_root_contains_canonical={}\n", + "MANAGED_PROXY_PATH_DIAG role={role} observation=host-after-failure absolute={} regular={:?} symlink={:?} exec_mode={:?} identity={:?} canonical_equals_logical={:?} declared_root_contains_canonical={:?} canonical_root_contains_canonical={:?}\n", path.is_absolute(), - metadata.as_ref().is_some_and(|info| info.is_file()), + metadata.as_ref().map(|info| info.is_file()), std::fs::symlink_metadata(path) .ok() - .is_some_and(|info| info.file_type().is_symlink()), - metadata - .as_ref() - .is_some_and(|info| info.mode() & 0o111 != 0), + .map(|info| info.file_type().is_symlink()), + metadata.as_ref().map(|info| info.mode() & 0o111 != 0), metadata.as_ref().map(|info| (info.dev(), info.ino())), - canonical.as_ref().is_some_and(|value| value == path), + canonical.as_ref().map(|value| value == path), canonical .as_ref() - .is_some_and(|value| value.starts_with(declared_root)), - canonical.as_ref().is_some_and(|value| canonical_root + .map(|value| value.starts_with(declared_root)), + canonical .as_ref() - .is_some_and(|root| value.starts_with(root))), + .zip(canonical_root.as_ref()) + .map(|(value, root)| value.starts_with(root)), ) } @@ -281,6 +281,8 @@ fn emit_managed_proxy_path_diagnostics(helper: &Path, declared_root: &Path) { &test_runner, declared_root, )); + } else { + facts.push_str("MANAGED_PROXY_PATH_DIAG role=test-runner observation=unavailable\n"); } match helper_elf_interpreter(helper) { Ok(Some(interpreter)) => facts.push_str(&managed_proxy_path_facts(