From cffdc3bdba4c0761b9ff6fc00066034ade594598 Mon Sep 17 00:00:00 2001 From: Gabor Gevay Date: Fri, 21 Aug 2026 20:10:09 +0200 Subject: [PATCH 1/2] ci: self-heal cargo caches that kill build scripts with a bare ENOENT A corrupted cached build-script binary fails with "failed to run custom build command" plus a bare "Error: No such file or directory (os error 2)", a signature clear-corrupted-cargo-target-dir did not recognize, so the corruption persisted across runs instead of being wiped and retried. On the single-agent merge-skew queue this took the check down for every PR until the agent was replaced by hand. Require both halves of the signature, with the ENOENT as the exact bare error line: the first half alone is any build-script bug, which a retry cannot fix, and a build script that fails on a missing file with an error context of its own reports the ENOENT in a "Caused by:" detail line instead. Mirror the signature in run_and_detect_retryable_build_failure, which the sync comments already tie to the shell script. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FJqg3FMKRiGYJ4owEwVAdC --- bin/clear-corrupted-cargo-target-dir | 17 +++++++++++++++++ misc/python/materialize/mzbuild.py | 18 +++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/bin/clear-corrupted-cargo-target-dir b/bin/clear-corrupted-cargo-target-dir index ae4bee910af63..0fcc9cdf25852 100755 --- a/bin/clear-corrupted-cargo-target-dir +++ b/bin/clear-corrupted-cargo-target-dir @@ -27,12 +27,29 @@ incremental_build_failure_pattern+="|signal: 11, SIGSEGV" registry_fetch_failure_pattern="unable to update registry" +# A cached build-script binary poisoned by cargo cache corruption dies with a +# bare ENOENT before doing any work. Both halves are required, and the error +# must be the exact bare line: "failed to run custom build command" alone is +# any build-script bug, which a retry cannot fix, and a build script that +# fails on a missing file with an error context of its own produces the ENOENT +# in a "Caused by:" detail line instead of the bare "Error:" line. +custom_build_script_enoent() { + grep -q "failed to run custom build command" "$1" \ + && grep -qE '^[[:space:]]*Error: No such file or directory \(os error 2\)[[:space:]]*$' "$1" +} + if grep -qE "$incremental_build_failure_pattern" "$1"; then echo "--- Detected incremental build failure, clearing cargo target directories" rm -rf target target-xcompile || true exit 199 fi +if custom_build_script_enoent "$1"; then + echo "--- Detected corrupted cached build script, clearing cargo target directories" + rm -rf target target-xcompile || true + exit 199 +fi + # Nothing on disk is corrupted, so keep the target directories. Wiping them # would turn a network blip into a full rebuild of the workspace, here and in # every later job that shares the agent's target directory. diff --git a/misc/python/materialize/mzbuild.py b/misc/python/materialize/mzbuild.py index 27264bc82909e..585f56f30fc38 100644 --- a/misc/python/materialize/mzbuild.py +++ b/misc/python/materialize/mzbuild.py @@ -142,7 +142,23 @@ def run_and_detect_retryable_build_failure( "unable to update registry", ] combined = stdout_contents + stderr_contents - if any(msg in combined for msg in incremental_build_failure_msgs): + # A cached build-script binary poisoned by cargo cache corruption dies + # with a bare ENOENT before doing any work. Both halves are required, + # and the error must be the exact bare line: "failed to run custom + # build command" alone is any build-script bug, which a retry cannot + # fix, and a build script that fails on a missing file with an error + # context of its own produces the ENOENT in a "Caused by:" detail line + # instead of the bare "Error:" line. + custom_build_script_enoent = ( + "failed to run custom build command" in combined + and any( + line.strip() == "Error: No such file or directory (os error 2)" + for line in combined.splitlines() + ) + ) + if custom_build_script_enoent or any( + msg in combined for msg in incremental_build_failure_msgs + ): raise RustIncrementalBuildFailure() if any(msg in combined for msg in registry_fetch_failure_msgs): raise CargoRegistryFetchFailure() From 66b2fce0e3be7a7dc2310d32fe743d56a13f6e44 Mon Sep 17 00:00:00 2001 From: Gabor Gevay Date: Fri, 21 Aug 2026 20:10:59 +0200 Subject: [PATCH 2/2] ci: annotate every self-healing cargo cache wipe The wipe-and-retry mechanism makes the build green, so if cache corruption ever became frequent its only symptom would be builds running cold over and over, attributed to nothing. Emit a warning annotation on every wipe, from the shell script and from both Python handlers, so each self-heal event stays visible on the build page and countable across builds, while the build still succeeds. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FJqg3FMKRiGYJ4owEwVAdC --- bin/clear-corrupted-cargo-target-dir | 16 ++++++++++++++++ ci/test/build.py | 3 ++- ci/test/cargo-test/mzcompose.py | 1 + misc/python/materialize/buildkite.py | 22 ++++++++++++++++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/bin/clear-corrupted-cargo-target-dir b/bin/clear-corrupted-cargo-target-dir index 0fcc9cdf25852..7b1819f3b4d33 100755 --- a/bin/clear-corrupted-cargo-target-dir +++ b/bin/clear-corrupted-cargo-target-dir @@ -38,14 +38,30 @@ custom_build_script_enoent() { && grep -qE '^[[:space:]]*Error: No such file or directory \(os error 2\)[[:space:]]*$' "$1" } +# Record the wipe as a build annotation. The wipe and retry make the build +# green, so without a durable trace the only symptom of recurring corruption +# is builds going cold repeatedly. Never fail the script over bookkeeping. +annotate_cache_wipe() { + if [ -n "${BUILDKITE_JOB_ID:-}" ] && command -v buildkite-agent >/dev/null 2>&1; then + buildkite-agent annotate \ + --style warning \ + --context "${BUILDKITE_JOB_ID}-cache-wipe" \ + "Corrupted cargo cache cleared; the build was retried + +Detected signature: $1. Frequent occurrences of this annotation mean builds keep running cold; investigate the corruption source rather than relying on the retry." || true + fi +} + if grep -qE "$incremental_build_failure_pattern" "$1"; then echo "--- Detected incremental build failure, clearing cargo target directories" + annotate_cache_wipe "incremental build failure" rm -rf target target-xcompile || true exit 199 fi if custom_build_script_enoent "$1"; then echo "--- Detected corrupted cached build script, clearing cargo target directories" + annotate_cache_wipe "corrupted cached build script (bare ENOENT)" rm -rf target target-xcompile || true exit 199 fi diff --git a/ci/test/build.py b/ci/test/build.py index 15f57424bee83..114c033dbf38a 100755 --- a/ci/test/build.py +++ b/ci/test/build.py @@ -15,7 +15,7 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from materialize import mzbuild, spawn, ui +from materialize import buildkite, mzbuild, spawn, ui from materialize.ci_util.upload_debug_symbols_to_s3 import ( DEBUGINFO_BINS, upload_debuginfo_to_s3, @@ -72,6 +72,7 @@ def main() -> None: print( "--- Detected incremental build failure, clearing cargo target directories" ) + buildkite.annotate_cache_wipe("incremental build failure") for dir in ["target", "target-xcompile"]: if os.path.exists(dir): shutil.rmtree(dir, ignore_errors=True) diff --git a/ci/test/cargo-test/mzcompose.py b/ci/test/cargo-test/mzcompose.py index c12ee385fdcad..5d914b7d84dd0 100644 --- a/ci/test/cargo-test/mzcompose.py +++ b/ci/test/cargo-test/mzcompose.py @@ -473,6 +473,7 @@ def worker() -> None: def _handle_incremental_build_failure() -> None: print("--- Detected incremental build failure, clearing cargo target directories") + buildkite.annotate_cache_wipe("incremental build failure") for dir in ["target", "target-xcompile"]: if os.path.exists(dir): shutil.rmtree(dir, ignore_errors=True) diff --git a/misc/python/materialize/buildkite.py b/misc/python/materialize/buildkite.py index cf29330ced23d..fee29224b4ecb 100644 --- a/misc/python/materialize/buildkite.py +++ b/misc/python/materialize/buildkite.py @@ -316,6 +316,28 @@ def add_annotation(style: str, title: str, content: str) -> None: add_annotation_raw(style, markdown) +def annotate_cache_wipe(signature: str) -> None: + """Record a self-healing cargo cache wipe as a build annotation. + + The wipe and retry make the build green, so without a durable trace the + only symptom of recurring corruption is builds going cold repeatedly. The + annotation keeps each wipe visible and countable. Never fails the caller: + the wipe matters more than its bookkeeping. + """ + if not os.getenv("BUILDKITE_JOB_ID"): + return + try: + add_annotation( + "warning", + "Corrupted cargo cache cleared; the build was retried", + f"Detected signature: {signature}. Frequent occurrences of this " + "annotation mean builds keep running cold; investigate the " + "corruption source rather than relying on the retry.", + ) + except Exception as e: + print(f"failed to annotate cache wipe: {e}") + + def get_job_url_from_build_url(build_url: str, build_job_id: str) -> str: return f"{build_url}#{build_job_id}"