ci: annotate every self-healing cargo cache wipe

bin/clear-corrupted-cargo-target-dir

@@ -27,8 +27,41 @@ incremental_build_failure_pattern+="|signal: 11, SIGSEGV"
 
 registry_fetch_failure_pattern="unable to update registry"
 
+# A cached build-script binary poisoned by cargo cache corruption dies with a
+# bare ENOENT before doing any work. Both halves are required, and the error
+# must be the exact bare line: "failed to run custom build command" alone is
+# any build-script bug, which a retry cannot fix, and a build script that
+# fails on a missing file with an error context of its own produces the ENOENT
+# in a "Caused by:" detail line instead of the bare "Error:" line.
+custom_build_script_enoent() {
+  grep -q "failed to run custom build command" "$1" \
+    && grep -qE '^[[:space:]]*Error: No such file or directory \(os error 2\)[[:space:]]*$' "$1"
+}
+
+# Record the wipe as a build annotation. The wipe and retry make the build
+# green, so without a durable trace the only symptom of recurring corruption
+# is builds going cold repeatedly. Never fail the script over bookkeeping.
+annotate_cache_wipe() {
+  if [ -n "${BUILDKITE_JOB_ID:-}" ] && command -v buildkite-agent >/dev/null 2>&1; then
+    buildkite-agent annotate \
+      --style warning \
+      --context "${BUILDKITE_JOB_ID}-cache-wipe" \
+      "Corrupted cargo cache cleared; the build was retried
+
+Detected signature: $1. Frequent occurrences of this annotation mean builds keep running cold; investigate the corruption source rather than relying on the retry." || true
+  fi
+}
+
 if grep -qE "$incremental_build_failure_pattern" "$1"; then
   echo "--- Detected incremental build failure, clearing cargo target directories"
+  annotate_cache_wipe "incremental build failure"
+  rm -rf target target-xcompile || true
+  exit 199
+fi
+
+if custom_build_script_enoent "$1"; then
+  echo "--- Detected corrupted cached build script, clearing cargo target directories"
+  annotate_cache_wipe "corrupted cached build script (bare ENOENT)"
   rm -rf target target-xcompile || true
   exit 199
 fi

ci/test/build.py

@@ -15,7 +15,7 @@
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 
-from materialize import mzbuild, spawn, ui
+from materialize import buildkite, mzbuild, spawn, ui
 from materialize.ci_util.upload_debug_symbols_to_s3 import (
     DEBUGINFO_BINS,
     upload_debuginfo_to_s3,
@@ -72,6 +72,7 @@ def main() -> None:
         print(
             "--- Detected incremental build failure, clearing cargo target directories"
         )
+        buildkite.annotate_cache_wipe("incremental build failure")
         for dir in ["target", "target-xcompile"]:
             if os.path.exists(dir):
                 shutil.rmtree(dir, ignore_errors=True)

ci/test/cargo-test/mzcompose.py

@@ -473,6 +473,7 @@ def worker() -> None:
 
 def _handle_incremental_build_failure() -> None:
     print("--- Detected incremental build failure, clearing cargo target directories")
+    buildkite.annotate_cache_wipe("incremental build failure")
     for dir in ["target", "target-xcompile"]:
         if os.path.exists(dir):
             shutil.rmtree(dir, ignore_errors=True)

misc/python/materialize/buildkite.py

@@ -316,6 +316,28 @@ def add_annotation(style: str, title: str, content: str) -> None:
     add_annotation_raw(style, markdown)
 
 
+def annotate_cache_wipe(signature: str) -> None:
+    """Record a self-healing cargo cache wipe as a build annotation.
+
+    The wipe and retry make the build green, so without a durable trace the
+    only symptom of recurring corruption is builds going cold repeatedly. The
+    annotation keeps each wipe visible and countable. Never fails the caller:
+    the wipe matters more than its bookkeeping.
+    """
+    if not os.getenv("BUILDKITE_JOB_ID"):
+        return
+    try:
+        add_annotation(
+            "warning",
+            "Corrupted cargo cache cleared; the build was retried",
+            f"Detected signature: {signature}. Frequent occurrences of this "
+            "annotation mean builds keep running cold; investigate the "
+            "corruption source rather than relying on the retry.",
+        )
+    except Exception as e:
+        print(f"failed to annotate cache wipe: {e}")
+
+
 def get_job_url_from_build_url(build_url: str, build_job_id: str) -> str:
     return f"{build_url}#{build_job_id}"
 

misc/python/materialize/mzbuild.py

@@ -142,7 +142,23 @@ def run_and_detect_retryable_build_failure(
             "unable to update registry",
         ]
         combined = stdout_contents + stderr_contents
-        if any(msg in combined for msg in incremental_build_failure_msgs):
+        # A cached build-script binary poisoned by cargo cache corruption dies
+        # with a bare ENOENT before doing any work. Both halves are required,
+        # and the error must be the exact bare line: "failed to run custom
+        # build command" alone is any build-script bug, which a retry cannot
+        # fix, and a build script that fails on a missing file with an error
+        # context of its own produces the ENOENT in a "Caused by:" detail line
+        # instead of the bare "Error:" line.
+        custom_build_script_enoent = (
+            "failed to run custom build command" in combined
+            and any(
+                line.strip() == "Error: No such file or directory (os error 2)"
+                for line in combined.splitlines()
+            )
+        )
+        if custom_build_script_enoent or any(
+            msg in combined for msg in incremental_build_failure_msgs
+        ):
             raise RustIncrementalBuildFailure()
         if any(msg in combined for msg in registry_fetch_failure_msgs):
             raise CargoRegistryFetchFailure()