diff --git a/bin/clear-corrupted-cargo-target-dir b/bin/clear-corrupted-cargo-target-dir index ae4bee910af63..7b1819f3b4d33 100755 --- a/bin/clear-corrupted-cargo-target-dir +++ b/bin/clear-corrupted-cargo-target-dir @@ -27,8 +27,41 @@ incremental_build_failure_pattern+="|signal: 11, SIGSEGV" registry_fetch_failure_pattern="unable to update registry" +# A cached build-script binary poisoned by cargo cache corruption dies with a +# bare ENOENT before doing any work. Both halves are required, and the error +# must be the exact bare line: "failed to run custom build command" alone is +# any build-script bug, which a retry cannot fix, and a build script that +# fails on a missing file with an error context of its own produces the ENOENT +# in a "Caused by:" detail line instead of the bare "Error:" line. +custom_build_script_enoent() { + grep -q "failed to run custom build command" "$1" \ + && grep -qE '^[[:space:]]*Error: No such file or directory \(os error 2\)[[:space:]]*$' "$1" +} + +# Record the wipe as a build annotation. The wipe and retry make the build +# green, so without a durable trace the only symptom of recurring corruption +# is builds going cold repeatedly. Never fail the script over bookkeeping. +annotate_cache_wipe() { + if [ -n "${BUILDKITE_JOB_ID:-}" ] && command -v buildkite-agent >/dev/null 2>&1; then + buildkite-agent annotate \ + --style warning \ + --context "${BUILDKITE_JOB_ID}-cache-wipe" \ + "Corrupted cargo cache cleared; the build was retried + +Detected signature: $1. Frequent occurrences of this annotation mean builds keep running cold; investigate the corruption source rather than relying on the retry." || true + fi +} + if grep -qE "$incremental_build_failure_pattern" "$1"; then echo "--- Detected incremental build failure, clearing cargo target directories" + annotate_cache_wipe "incremental build failure" + rm -rf target target-xcompile || true + exit 199 +fi + +if custom_build_script_enoent "$1"; then + echo "--- Detected corrupted cached build script, clearing cargo target directories" + annotate_cache_wipe "corrupted cached build script (bare ENOENT)" rm -rf target target-xcompile || true exit 199 fi diff --git a/ci/test/build.py b/ci/test/build.py index 15f57424bee83..114c033dbf38a 100755 --- a/ci/test/build.py +++ b/ci/test/build.py @@ -15,7 +15,7 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from materialize import mzbuild, spawn, ui +from materialize import buildkite, mzbuild, spawn, ui from materialize.ci_util.upload_debug_symbols_to_s3 import ( DEBUGINFO_BINS, upload_debuginfo_to_s3, @@ -72,6 +72,7 @@ def main() -> None: print( "--- Detected incremental build failure, clearing cargo target directories" ) + buildkite.annotate_cache_wipe("incremental build failure") for dir in ["target", "target-xcompile"]: if os.path.exists(dir): shutil.rmtree(dir, ignore_errors=True) diff --git a/ci/test/cargo-test/mzcompose.py b/ci/test/cargo-test/mzcompose.py index c12ee385fdcad..5d914b7d84dd0 100644 --- a/ci/test/cargo-test/mzcompose.py +++ b/ci/test/cargo-test/mzcompose.py @@ -473,6 +473,7 @@ def worker() -> None: def _handle_incremental_build_failure() -> None: print("--- Detected incremental build failure, clearing cargo target directories") + buildkite.annotate_cache_wipe("incremental build failure") for dir in ["target", "target-xcompile"]: if os.path.exists(dir): shutil.rmtree(dir, ignore_errors=True) diff --git a/misc/python/materialize/buildkite.py b/misc/python/materialize/buildkite.py index cf29330ced23d..fee29224b4ecb 100644 --- a/misc/python/materialize/buildkite.py +++ b/misc/python/materialize/buildkite.py @@ -316,6 +316,28 @@ def add_annotation(style: str, title: str, content: str) -> None: add_annotation_raw(style, markdown) +def annotate_cache_wipe(signature: str) -> None: + """Record a self-healing cargo cache wipe as a build annotation. + + The wipe and retry make the build green, so without a durable trace the + only symptom of recurring corruption is builds going cold repeatedly. The + annotation keeps each wipe visible and countable. Never fails the caller: + the wipe matters more than its bookkeeping. + """ + if not os.getenv("BUILDKITE_JOB_ID"): + return + try: + add_annotation( + "warning", + "Corrupted cargo cache cleared; the build was retried", + f"Detected signature: {signature}. Frequent occurrences of this " + "annotation mean builds keep running cold; investigate the " + "corruption source rather than relying on the retry.", + ) + except Exception as e: + print(f"failed to annotate cache wipe: {e}") + + def get_job_url_from_build_url(build_url: str, build_job_id: str) -> str: return f"{build_url}#{build_job_id}" diff --git a/misc/python/materialize/mzbuild.py b/misc/python/materialize/mzbuild.py index 27264bc82909e..585f56f30fc38 100644 --- a/misc/python/materialize/mzbuild.py +++ b/misc/python/materialize/mzbuild.py @@ -142,7 +142,23 @@ def run_and_detect_retryable_build_failure( "unable to update registry", ] combined = stdout_contents + stderr_contents - if any(msg in combined for msg in incremental_build_failure_msgs): + # A cached build-script binary poisoned by cargo cache corruption dies + # with a bare ENOENT before doing any work. Both halves are required, + # and the error must be the exact bare line: "failed to run custom + # build command" alone is any build-script bug, which a retry cannot + # fix, and a build script that fails on a missing file with an error + # context of its own produces the ENOENT in a "Caused by:" detail line + # instead of the bare "Error:" line. + custom_build_script_enoent = ( + "failed to run custom build command" in combined + and any( + line.strip() == "Error: No such file or directory (os error 2)" + for line in combined.splitlines() + ) + ) + if custom_build_script_enoent or any( + msg in combined for msg in incremental_build_failure_msgs + ): raise RustIncrementalBuildFailure() if any(msg in combined for msg in registry_fetch_failure_msgs): raise CargoRegistryFetchFailure()