fix(upgrade): preserve preparation-failure evidence and complete the upgrade matrix

.github/workflows/rustfs-upgrade-test.yml

@@ -74,7 +74,7 @@ defaults:
 env:
   RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
   RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
-  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
   RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -133,7 +133,7 @@ jobs:
         if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
         run: |
           set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002 vm003}"
           SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
           for node in "${NODES[@]}"; do
             ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
@@ -174,7 +174,7 @@ jobs:
           TO_VERSION='${{ inputs.to_version }}'
           TOPOLOGY='${{ inputs.topology }}'
           BACKENDS='${{ inputs.backends }}'
-          ARGS=(-y --log-file "${LOG_FILE}")
+          ARGS=(-y --log-file "${LOG_FILE}" --artifacts-dir "${FUNCTIONAL_ARTIFACTS_DIR}/upgrade")
           if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
             ARGS+=(--all-topologies)
           else
@@ -246,19 +246,23 @@ jobs:
           fi
           CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
           MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md"
+          CASE_LOG="${FUNCTIONAL_ARTIFACTS_DIR}/upgrade/events.log"
+          [ -f "${CASE_LOG}" ] || CASE_LOG="${LOG_FILE}"
           CASE_RESULT=success
-          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
+          python3 scripts/functional_case_report.py "${CASE_LOG}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
           RESULT=failure
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
 
           # Report harness completion here; the final verdict checks every
           # case result after diagnostics and cleanup have been retained.
-          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING|NOT_RUN) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_PASS="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| PASS \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_NOT_RUN="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (NOT_RUN|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
-          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          CASES_PASS=$(( ${CASES_PASS:-0} + 0 )); CASES_NOT_RUN=$(( ${CASES_NOT_RUN:-0} + 0 ))
           HARNESS_OK=0
           if [ '${{ steps.test.outcome }}' = 'success' ]; then
             HARNESS_OK=1
@@ -285,7 +289,7 @@ jobs:
               cat "${CASE_TABLE}"
               echo ""
             fi
-            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo "- 用例结果：${CASES_PASS} 通过，${CASES_FAIL} 断言失败，${CASES_NOT_RUN} 未完成；准备、升级及恢复状态见矩阵。"
             echo ""
             if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
@@ -401,6 +405,7 @@ jobs:
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/matrix.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/upgrade/
           if-no-files-found: error
           retention-days: 3
 
@@ -410,7 +415,7 @@ jobs:
         if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002 vm003}"
           SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
           for node in "${NODES[@]}"; do
             ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
@@ -431,7 +436,14 @@ jobs:
         env:
           FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-upgrade.json
           CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
-        run: python3 scripts/functional_environment.py finish --suite upgrade --cleanup-outcome "$CLEANUP_OUTCOME"
+        run: |
+          # The outer node cleanup cannot prove Vault cleanup or that the
+          # interrupted suite cleaned every resource it touched.
+          if ! python3 scripts/upgrade_evidence.py check-cleanup \
+            --state-file "${FUNCTIONAL_ARTIFACTS_DIR}/upgrade/fleet-cleanup.state"; then
+            CLEANUP_OUTCOME=failure
+          fi
+          python3 scripts/functional_environment.py finish --suite upgrade --cleanup-outcome "$CLEANUP_OUTCOME"
 
       - name: Retain environment ownership and cleanup receipt
         if: ${{ always() }}

.github/workflows/upgrade-script-tests.yml

@@ -0,0 +1,50 @@
+name: Upgrade Script Tests
+
+on:
+  pull_request:
+    paths:
+      - rustfs-upgrade-test.sh
+      - scripts/upgrade_evidence.py
+      - scripts/functional_case_report.py
+      - scripts/issue_manager.py
+      - scripts/lib/**
+      - test_rustfs_upgrade.py
+      - test_issue_manager.py
+      - scripts/functional_environment.py
+      - .github/workflows/rustfs-upgrade-test.yml
+      - .github/workflows/upgrade-script-tests.yml
+  push:
+    branches: [main]
+    paths:
+      - rustfs-upgrade-test.sh
+      - scripts/upgrade_evidence.py
+      - scripts/functional_case_report.py
+      - scripts/issue_manager.py
+      - scripts/lib/**
+      - test_rustfs_upgrade.py
+      - test_issue_manager.py
+      - scripts/functional_environment.py
+      - .github/workflows/rustfs-upgrade-test.yml
+      - .github/workflows/upgrade-script-tests.yml
+
+permissions:
+  contents: read
+
+jobs:
+  upgrade-script-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Install protocol fixture client
+        run: |
+          python3 -m venv "${RUNNER_TEMP}/upgrade-tests"
+          "${RUNNER_TEMP}/upgrade-tests/bin/pip" install awscli==1.45.19
+          echo "${RUNNER_TEMP}/upgrade-tests/bin" >> "${GITHUB_PATH}"
+          command -v jq
+      - name: Validate upgrade suite and evidence handling
+        run: |
+          bash -n rustfs-upgrade-test.sh
+          python3 -B -m unittest -v test_rustfs_upgrade.py test_sensitive_diagnostics.py test_issue_manager.py

docs/upgrade-test-evidence.md

@@ -0,0 +1,41 @@
+# 升级套件的种子与结果证据
+
+`rustfs-upgrade-test.sh` 在每个拓扑和 KMS 后端组合上依次执行旧版准备、原地升级、18 条升级后断言、诊断归档和环境清理。默认六组，共 108 次用例执行。
+
+100MiB 种子固定写入两个对象，避免按时间生成大量对象后再裁剪。1KiB 和 1MiB 种子保留 Warp 生成方式，分别保留五个对象。升级前读回种子，记录完整键清单、字节数和 SHA-256；升级后逐对象比较清单和内容，并保留 HEAD/GET ETag 检查。
+
+裁剪请求每批最多 1000 个键。AWS CLI 退出成功仍需检查响应的 `Errors`，发现逐对象错误立即停止准备，不自动重试或把错误当作成功。每批保存请求键、CLI 退出码、可解析的 HTTP 状态、请求 ID 和安全响应。顶层 S3 错误仅保留 Code、Message 和 RequestId；签名请求、原始 CLI debug、凭据和 golden.tsv 留在私有临时目录，退出时删除。
+
+通过 `--artifacts-dir <新目录>` 指定证据目录，或使用 `RUSTFS_UPGRADE_ARTIFACTS_DIR`。目录必须尚不存在，避免复用旧结果。未指定时默认使用 `<log-file>.artifacts`，无日志路径时使用当前目录中的 UTC 时间路径。
+
+证据目录包含：
+
+- `events.log`：同步写入的逐项用例和阶段结果，不依赖 tee 完成时序。
+- `plan.tsv`：依赖检查前原子保存的选定组合及其全部用例 ID。解析器对照计划检查每组的用例、终态、清理及诊断记录，缺失或重复都会失败；缺失用例补为 NOT_RUN。
+- `fleet-cleanup.state`：触及环境前标记 PENDING，所有目标节点及 Vault 清理验证完成后才标记 PASS。缺失、PENDING、FAIL 或 RETAINED 均不能授权 Actions 释放共享环境。
+- `cases.md`、`matrix.md`、`results.json`：所有选定组合的逐项结果和结构化汇总。
+- `<拓扑>-<后端>/`：种子清单、删除响应、逐项诊断、Warp 输出及节点文件日志和 journal。节点文件日志最多保留每文件末尾 4MiB、每节点合计 16MiB，截断会标明。
+
+报告状态：
+
+| 状态 | 含义 |
+| --- | --- |
+| PASS | 升级后断言通过，诊断归档成功，环境已清理或显式保留 |
+| PREPARE_FAILED | 旧版部署、健康检查或种子准备失败，尚未进入升级 |
+| UPGRADE_FAILED | 原地升级或升级后健康检查失败 |
+| VERIFY_FAILED | 已进入新版验证，至少一个断言失败 |
+| INTERRUPTED | 收到 INT/TERM 中断 |
+| INCOMPLETE | 进程或日志中断，计划中的终态证据缺失 |
+| DIAGNOSTICS_FAILED | 诊断证据未完整归档 |
+| CLEANUP_FAILED | 环境清理未完成 |
+| NOT_RUN | 该组或用例未执行完成，不能作为通过证据 |
+
+准备失败和中断都会补齐未完成的用例，先归档再清理当前组，并将后续组标记为未执行。普通断言失败在清理成功后继续其他组合；诊断或清理失败则停止后续组。主失败原因与清理/诊断结果分别保留。`--keep-env` 显式保留环境，仍归档证据并标记清理为 `RETAINED`。
+
+依赖检查失败会保留整份计划、准备失败记录及所有未执行用例，不触发环境清理。Python 不可用时仍保留 Bash 写入的事件供后续解析。诊断收集中断保留失败状态，退出处理不得把不完整归档改为通过。dry-run 不打开或覆盖指定日志文件。
+
+Actions 的隔离标记、套件目标节点和外层清理使用同一份 `RUSTFS_NODES`，默认包含 vm000–vm003。释放环境同时要求外层清理成功与套件清理证据通过，任一失败均保留隔离所有权，需在共享锁下完成恢复再解除。
+
+例如三组 local 验证通过后，vault-kv2 第一组种子准备失败，报告应为 54 PASS、54 NOT_RUN，矩阵包含 `PREPARE_FAILED`。这证明本轮升级矩阵未完成，不能据此判定新版 Vault 兼容性失败，也不能自动关闭历史产品 Issue。
+
+GitHub Actions 上传上述安全证据目录。独立的 `Upgrade Script Tests` 使用本地协议 fixture 和模拟环境边界验证脚本，包含真实 AWS CLI 的 HTTP 200 逐对象错误、顶层 503 和批量删除分片。这些检查不连接 VM，不代表产品实机复测。