From 8d96dc9ff46e7e2156c610ce5595c0228f87e59b Mon Sep 17 00:00:00 2001 From: Randolph Settgast Date: Wed, 12 Aug 2026 17:01:12 -0700 Subject: [PATCH 1/2] Add hardened Azure HBv4 MPI TPL image lifecycle Provision ephemeral HB176rs_v4 JIT builder and tester runners for OpenMPI and MPICH, validate the fixed CPU and NVMe contract, and add fail-closed cleanup and janitor automation. Build Zen 4 TPL images from source with pinned Spack revisions, a source-only mirror, bounded fetch and bootstrap retries, and a shared 176-job build pool. Capture MPI, HDF5, and target evidence; validate candidates offline on fresh runners; promote exact ACR digests; and publish immutable pull instructions. Harden workflow authorization and artifact contracts with Azure OIDC, GitHub App client IDs, Node 24 action pins, trusted sparse checkouts, and credential-free regression fixtures. --- .../_docker_build_tpls_hbv4_provider.yml | 843 ++++++++++++++++ .github/workflows/azure-janitor.yml | 76 ++ .github/workflows/docker_build_tpls_hbv4.yml | 220 ++++ .uberenv_config.json | 8 +- .../scripts/finalize-hbv4-candidate.sh | 143 +++ ci/azure/actions/cleanup-runner/action.yml | 627 ++++++++++++ .../actions/provision-hbv4-runner/action.yml | 562 +++++++++++ ci/azure/scripts/janitor-hbv4.sh | 484 +++++++++ ci/azure/scripts/probe-runner-bootstrap.sh | 342 +++++++ ci/azure/scripts/render-runner-bootstrap.py | 102 ++ ci/azure/scripts/runner-bootstrap.sh | 954 ++++++++++++++++++ ci/azure/scripts/runner-names.sh | 72 ++ ci/azure/scripts/setup-hbv4-local-nvme.sh | 523 ++++++++++ ci/azure/scripts/validate-created-hbv4-vm.sh | 101 ++ ci/azure/tests/fixtures/hbv4-vm-show.json | 47 + .../tests/test-cleanup-runner-registration.sh | 167 +++ .../tests/test-created-hbv4-vm-contract.sh | 70 ++ ci/azure/tests/test-hbv4-build-resilience.sh | 134 +++ ci/azure/tests/test-hbv4-local-nvme.sh | 454 +++++++++ ci/azure/tests/test-hbv4-workflow-contract.sh | 370 +++++++ .../tests/test-render-runner-bootstrap.py | 220 ++++ ci/azure/tests/test-runner-bootstrap-probe.sh | 247 +++++ ci/azure/tests/test-runner-names.sh | 65 ++ docker/install-mpi4py.sh | 57 ++ docker/install-mpich.sh | 52 + docker/install-openmpi.sh | 54 + docker/tpl-ubuntu.Dockerfile | 201 +++- docker/ubuntu-hbv4-mpich-spack.yaml | 250 +++++ docker/ubuntu-hbv4-openmpi-spack.yaml | 250 +++++ docker/ubuntu-spack.yaml | 32 +- scripts/docker-build.sh | 86 +- scripts/hbv4/README.md | 66 ++ scripts/hbv4/capture-mpi-wrapper-show.sh | 33 + scripts/hbv4/collect-build-evidence.sh | 172 ++++ scripts/hbv4/install-spack-from-source.sh | 64 ++ scripts/hbv4/parallel_hdf5_shared.c | 129 +++ scripts/hbv4/validate-hbv4-tpls | 430 ++++++++ scripts/hbv4/validate-host.sh | 86 ++ 38 files changed, 8730 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/_docker_build_tpls_hbv4_provider.yml create mode 100644 .github/workflows/azure-janitor.yml create mode 100644 .github/workflows/docker_build_tpls_hbv4.yml create mode 100755 ci/azure-images/scripts/finalize-hbv4-candidate.sh create mode 100644 ci/azure/actions/cleanup-runner/action.yml create mode 100644 ci/azure/actions/provision-hbv4-runner/action.yml create mode 100755 ci/azure/scripts/janitor-hbv4.sh create mode 100755 ci/azure/scripts/probe-runner-bootstrap.sh create mode 100755 ci/azure/scripts/render-runner-bootstrap.py create mode 100755 ci/azure/scripts/runner-bootstrap.sh create mode 100755 ci/azure/scripts/runner-names.sh create mode 100755 ci/azure/scripts/setup-hbv4-local-nvme.sh create mode 100755 ci/azure/scripts/validate-created-hbv4-vm.sh create mode 100644 ci/azure/tests/fixtures/hbv4-vm-show.json create mode 100755 ci/azure/tests/test-cleanup-runner-registration.sh create mode 100755 ci/azure/tests/test-created-hbv4-vm-contract.sh create mode 100755 ci/azure/tests/test-hbv4-build-resilience.sh create mode 100755 ci/azure/tests/test-hbv4-local-nvme.sh create mode 100755 ci/azure/tests/test-hbv4-workflow-contract.sh create mode 100755 ci/azure/tests/test-render-runner-bootstrap.py create mode 100755 ci/azure/tests/test-runner-bootstrap-probe.sh create mode 100755 ci/azure/tests/test-runner-names.sh create mode 100755 docker/install-mpi4py.sh create mode 100755 docker/install-mpich.sh create mode 100755 docker/install-openmpi.sh create mode 100644 docker/ubuntu-hbv4-mpich-spack.yaml create mode 100644 docker/ubuntu-hbv4-openmpi-spack.yaml create mode 100644 scripts/hbv4/README.md create mode 100755 scripts/hbv4/capture-mpi-wrapper-show.sh create mode 100755 scripts/hbv4/collect-build-evidence.sh create mode 100755 scripts/hbv4/install-spack-from-source.sh create mode 100644 scripts/hbv4/parallel_hdf5_shared.c create mode 100755 scripts/hbv4/validate-hbv4-tpls create mode 100755 scripts/hbv4/validate-host.sh diff --git a/.github/workflows/_docker_build_tpls_hbv4_provider.yml b/.github/workflows/_docker_build_tpls_hbv4_provider.yml new file mode 100644 index 00000000..c90827cf --- /dev/null +++ b/.github/workflows/_docker_build_tpls_hbv4_provider.yml @@ -0,0 +1,843 @@ +name: HBv4 TPL provider lifecycle + +on: + workflow_call: + inputs: + provider: + required: true + type: string + source_sha: + required: true + type: string + control_sha: + required: true + type: string + event_name: + required: true + type: string + event_ref: + required: true + type: string + pr_number: + required: false + default: '' + type: string + +permissions: {} + +# A provider owns at most one paid VM across all runs. Builder and tester are +# also sequenced explicitly inside this workflow. +concurrency: + group: hbv4-tpl-${{ github.repository }}-${{ inputs.provider }} + cancel-in-progress: false + +env: + DOCKER_BASE_IMAGE: geosx/ubuntu:24.04-gcc13-1c3c049b3f629d9d44838656fd306b2a0c04c9e8 + TPL_DOCKERFILE: docker/tpl-ubuntu.Dockerfile + SPEC: "~pygeosx ~docs %gcc-13" + +jobs: + provision_builder: + name: Provision ${{ inputs.provider }} builder + runs-on: ubuntu-24.04 + timeout-minutes: 35 + environment: azure-ci + permissions: + contents: read + id-token: write + pull-requests: read + outputs: + runner_label: ${{ steps.provision.outputs.runner_label }} + vm_name: ${{ steps.provision.outputs.vm_name }} + runner_id: ${{ steps.provision.outputs.runner_id }} + provision_run_attempt: ${{ steps.provision.outputs.provision_run_attempt }} + steps: + - name: Checkout trusted runner control plane + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.control_sha }} + fetch-depth: 1 + persist-credentials: false + sparse-checkout: | + ci/azure + scripts/hbv4/validate-host.sh + scripts/hbv4/validate-hbv4-tpls + scripts/hbv4/parallel_hdf5_shared.c + sparse-checkout-cone-mode: false + + # Match the GEOS lifecycle's token boundary: source authorization uses + # the job's read-only GITHUB_TOKEN; the GitHub App minted inside the + # provision action is reserved exclusively for runner administration. + - name: Reauthorize live source before paid builder provisioning + env: + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ inputs.event_name }} + EVENT_REF: ${{ inputs.event_ref }} + PR_NUMBER: ${{ inputs.pr_number }} + SOURCE_SHA: ${{ inputs.source_sha }} + run: | + set -euo pipefail + case "${EVENT_NAME}" in + pull_request_target) + [[ "${PR_NUMBER}" =~ ^[1-9][0-9]*$ ]] + [[ "${EVENT_REF}" == "refs/pull/${PR_NUMBER}/head" ]] + live_pr="$(gh api -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" + jq -e \ + --arg repository "${GITHUB_REPOSITORY}" \ + --arg sha "${SOURCE_SHA}" \ + --argjson number "${PR_NUMBER}" ' + .number == $number and .state == "open" and + .head.repo.full_name == $repository and .head.sha == $sha and + any(.labels[]?; .name == "ci: build HBv4 TPL") + ' <<< "${live_pr}" >/dev/null || { + echo '::error::Live PR authorization changed before paid builder provisioning.' + exit 1 + } + ;; + workflow_dispatch) + [[ -z "${PR_NUMBER}" ]] + live_commit="$(gh api -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/commits/${SOURCE_SHA}")" + [[ "$(jq -r '.sha // empty' <<< "${live_commit}")" == "${SOURCE_SHA}" ]] || { + echo '::error::Manual source commit no longer resolves exactly.' + exit 1 + } + ;; + *) exit 2 ;; + esac + + - name: Provision builder JIT runner + id: provision + uses: ./ci/azure/actions/provision-hbv4-runner + with: + provider: ${{ inputs.provider }} + runner_role: builder + event_name: ${{ inputs.event_name }} + event_ref: ${{ inputs.event_ref }} + source_sha: ${{ inputs.source_sha }} + pr_number: ${{ inputs.pr_number }} + azure_client_id: ${{ vars.AZURE_CLIENT_ID }} + azure_tenant_id: ${{ vars.AZURE_TENANT_ID }} + azure_subscription_id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + foundation_resource_group: ${{ vars.AZURE_FOUNDATION_RESOURCE_GROUP }} + runner_resource_group: ${{ vars.AZURE_RUNNER_RESOURCE_GROUP }} + runner_vm_uami_client_id: ${{ vars.AZURE_RUNNER_VM_UAMI_CLIENT_ID }} + acr_name: ${{ vars.AZURE_ACR_NAME }} + # Match the reviewed earthflow-sim/GEOS procurement contract: the + # repository-level Default runner group has stable ID 1. + runner_group_id: '1' + runner_app_client_id: ${{ vars.AZURE_RUNNER_APP_CLIENT_ID }} + runner_app_private_key: ${{ secrets.AZURE_RUNNER_APP_PRIVATE_KEY }} + + build_candidate: + name: Build ${{ inputs.provider }} candidate + needs: provision_builder + if: ${{ needs.provision_builder.result == 'success' && needs.provision_builder.outputs.provision_run_attempt == format('{0}', github.run_attempt) }} + runs-on: ${{ needs.provision_builder.outputs.runner_label }} + timeout-minutes: 330 + environment: azure-ci + permissions: + contents: read + id-token: write + pull-requests: read + outputs: + candidate_repository: ${{ steps.build.outputs.candidate_repository }} + candidate_digest: ${{ steps.publish.outputs.candidate_digest }} + candidate_digest_reference: ${{ steps.publish.outputs.candidate_digest_reference }} + base_digest: ${{ steps.build.outputs.base_digest }} + steps: + - name: Checkout exact source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.source_sha }} + fetch-depth: 1 + submodules: recursive + lfs: false + persist-credentials: false + + - name: Validate fixed HBv4 host contract + run: >- + /opt/geos-ci/scripts/hbv4/validate-host.sh + --instance-type Standard_HB176rs_v4 + --compiler-flags 'target=zen4 -march=native -mtune=native' + --writable-path "${GITHUB_WORKSPACE}" + + - name: Build without cache + id: build + env: + ACR_NAME: ${{ vars.AZURE_ACR_NAME }} + PROVIDER: ${{ inputs.provider }} + SOURCE_SHA: ${{ inputs.source_sha }} + TMPDIR: /mnt/hbv4-local/container-tmp + DOCKER_BUILDKIT: '1' + run: | + set -euo pipefail + case "${PROVIDER}" in openmpi|mpich) ;; *) exit 2 ;; esac + [[ "${SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]] + [[ "${ACR_NAME}" =~ ^[a-z0-9]{5,50}$ ]] + + docker pull "${DOCKER_BASE_IMAGE}" + mapfile -t base_repo_digests < <(docker image inspect \ + --format '{{range .RepoDigests}}{{println .}}{{end}}' \ + "${DOCKER_BASE_IMAGE}" | grep '^geosx/ubuntu@sha256:' | sort -u) + if (( ${#base_repo_digests[@]} != 1 )) || + ! [[ "${base_repo_digests[0]}" =~ ^geosx/ubuntu@sha256:[0-9a-f]{64}$ ]]; then + echo '::error::Pinned base tag did not resolve to exactly one immutable geosx/ubuntu digest.' + exit 1 + fi + immutable_base_image="${base_repo_digests[0]}" + + case "${PROVIDER}" in + openmpi) + mpi_prefix='/opt/openmpi/5.0.10' + mpiexec_path='/opt/openmpi/5.0.10/bin/mpirun' + ;; + mpich) + mpi_prefix='/opt/mpich/5.0.1' + mpiexec_path='/opt/mpich/5.0.1/bin/mpiexec' + ;; + esac + # Replace source-controlled ignore rules before sending the context to + # Docker. Checkout metadata and runner work/temporary artifacts are + # never build inputs. + printf '%s\n' \ + '.git' \ + '.git/**' \ + '**/.git' \ + '**/.git/**' \ + '_work' \ + '_work/**' \ + '_temp' \ + '_temp/**' \ + 'hbv4-evidence' \ + 'hbv4-evidence/**' \ + > .dockerignore + + candidate_repository="geos/hbv4-tpl-candidates/${PROVIDER}/run-${GITHUB_RUN_ID}-attempt-${GITHUB_RUN_ATTEMPT}" + candidate_reference="${ACR_NAME}.azurecr.io/${candidate_repository}:candidate" + install_dir="/opt/GEOS/GEOS_TPL-hbv4-${PROVIDER}-${SOURCE_SHA:0:12}" + + docker build \ + --no-cache \ + --pull \ + --progress=plain \ + --build-arg "DOCKER_BASE_IMAGE=${immutable_base_image}" \ + --build-arg 'GCC_VERSION=13' \ + --build-arg 'HOST_CONFIG=host-configs/environment.cmake' \ + --build-arg "INSTALL_DIR=${install_dir}" \ + --build-arg "SPEC=${SPEC}" \ + --build-arg "MPI_PROVIDER=${PROVIDER}" \ + --build-arg "MPI_PREFIX=${mpi_prefix}" \ + --build-arg "MPIEXEC_PATH=${mpiexec_path}" \ + --build-arg "SPACK_ENV_FILE=docker/ubuntu-hbv4-${PROVIDER}-spack.yaml" \ + --build-arg 'HBV4_BUILD=1' \ + --build-arg "SOURCE_SHA=${SOURCE_SHA}" \ + --build-arg 'FINAL_IMAGE_VARIANT=hbv4' \ + --label "org.opencontainers.image.revision=${SOURCE_SHA}" \ + --label "org.opencontainers.image.base.name=${immutable_base_image}" \ + --tag "${candidate_reference}" \ + --file "${TPL_DOCKERFILE}" \ + . + + { + echo "candidate_repository=${candidate_repository}" + echo "candidate_reference=${candidate_reference}" + echo "base_digest=${immutable_base_image}" + } >> "${GITHUB_OUTPUT}" + + - name: Reauthorize live source before registry credentials + env: + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ inputs.event_name }} + PR_NUMBER: ${{ inputs.pr_number }} + SOURCE_SHA: ${{ inputs.source_sha }} + run: | + set -euo pipefail + case "${EVENT_NAME}" in + pull_request_target) + live_pr="$(gh api -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" + jq -e \ + --arg repository "${GITHUB_REPOSITORY}" \ + --arg sha "${SOURCE_SHA}" \ + --argjson number "${PR_NUMBER}" ' + .number == $number and .state == "open" and + .head.repo.full_name == $repository and .head.sha == $sha and + any(.labels[]?; .name == "ci: build HBv4 TPL") + ' <<< "${live_pr}" >/dev/null || { + echo '::error::PR head/label authorization changed during the build.' + exit 1 + } + ;; + workflow_dispatch) + live_commit="$(gh api -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/commits/${SOURCE_SHA}")" + [[ "$(jq -r '.sha // empty' <<< "${live_commit}")" == "${SOURCE_SHA}" ]] || { + echo '::error::Manual source commit no longer resolves exactly.' + exit 1 + } + ;; + *) exit 2 ;; + esac + + # Acquire a fresh OIDC session only after the potentially multi-hour + # source build. The ACR Docker token is short-lived and must not age while + # Spack compiles the TPL graph. + - name: Login to Azure for candidate publication + uses: azure/login@f5d393ae46f8fde4be8b75f32e3fc50e654ad0ca # v3.0.1 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + - name: Publish candidate and capture build evidence + id: publish + env: + ACR_NAME: ${{ vars.AZURE_ACR_NAME }} + PROVIDER: ${{ inputs.provider }} + SOURCE_SHA: ${{ inputs.source_sha }} + CANDIDATE_REPOSITORY: ${{ steps.build.outputs.candidate_repository }} + CANDIDATE_REFERENCE: ${{ steps.build.outputs.candidate_reference }} + IMMUTABLE_BASE_IMAGE: ${{ steps.build.outputs.base_digest }} + run: | + set -euo pipefail + case "${PROVIDER}" in openmpi|mpich) ;; *) exit 2 ;; esac + [[ "${SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]] + [[ "${ACR_NAME}" =~ ^[a-z0-9]{5,50}$ ]] + expected_repository="geos/hbv4-tpl-candidates/${PROVIDER}/run-${GITHUB_RUN_ID}-attempt-${GITHUB_RUN_ATTEMPT}" + [[ "${CANDIDATE_REPOSITORY}" == "${expected_repository}" ]] + [[ "${CANDIDATE_REFERENCE}" == "${ACR_NAME}.azurecr.io/${CANDIDATE_REPOSITORY}:candidate" ]] + [[ "${IMMUTABLE_BASE_IMAGE}" =~ ^geosx/ubuntu@sha256:[0-9a-f]{64}$ ]] + + # The checked-out source never receives Azure credentials as a build + # argument, secret mount, environment variable, or Docker context file. + # Refresh the registry session immediately before the only push. + az acr login --name "${ACR_NAME}" --only-show-errors >/dev/null + docker push "${CANDIDATE_REFERENCE}" + candidate_digest="$(az acr repository show \ + --name "${ACR_NAME}" \ + --image "${CANDIDATE_REPOSITORY}:candidate" \ + --query digest --output tsv --only-show-errors)" + if ! [[ "${candidate_digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo '::error::ACR did not return a canonical candidate digest.' + exit 1 + fi + candidate_digest_reference="${ACR_NAME}.azurecr.io/${CANDIDATE_REPOSITORY}@${candidate_digest}" + { + echo "candidate_digest=${candidate_digest}" + echo "candidate_digest_reference=${candidate_digest_reference}" + } >> "${GITHUB_OUTPUT}" + + evidence_dir="${RUNNER_TEMP}/hbv4-evidence" + install -d -m 0700 "${evidence_dir}" + embedded_evidence="${evidence_dir}/embedded-build-evidence" + install -d -m 0700 "${embedded_evidence}" + evidence_container="$(docker create "${CANDIDATE_REFERENCE}" /bin/true)" + trap 'docker rm -f "${evidence_container}" >/dev/null 2>&1 || true' EXIT + docker cp "${evidence_container}:/opt/GEOS/hbv4-build-evidence/." "${embedded_evidence}/" + docker rm "${evidence_container}" >/dev/null + trap - EXIT + (cd "${embedded_evidence}" && sha256sum --check --strict checksums.sha256) + docker image inspect "${CANDIDATE_REFERENCE}" | jq '.[0] | { + Id, RepoDigests, Architecture, Os, Size, + Config:{Labels:.Config.Labels,Entrypoint:.Config.Entrypoint,Cmd:.Config.Cmd} + }' > "${evidence_dir}/image-inspect.json" + docker run --rm --network none --entrypoint cat "${CANDIDATE_REFERENCE}" \ + /opt/GEOS/mpi-provider.json 2>&1 | tee "${evidence_dir}/mpi-provider.json" + jq -e \ + --arg provider "${PROVIDER}" \ + --arg tpl_source_sha "${SOURCE_SHA}" \ + --arg base_digest "${IMMUTABLE_BASE_IMAGE}" ' + .provider == $provider and .tpl_source_sha == $tpl_source_sha and + .base_digest == $base_digest + ' "${evidence_dir}/mpi-provider.json" >/dev/null + docker run --rm --network none --entrypoint sha256sum "${CANDIDATE_REFERENCE}" \ + /opt/GEOS/bin/validate-hbv4-tpls > "${evidence_dir}/embedded-validator.sha256" + jq -n \ + --arg provider "${PROVIDER}" \ + --arg tpl_source_sha "${SOURCE_SHA}" \ + --arg base_digest "${IMMUTABLE_BASE_IMAGE}" \ + --arg candidate_repository "${CANDIDATE_REPOSITORY}" \ + --arg candidate_digest "${candidate_digest}" \ + '{provider:$provider,tpl_source_sha:$tpl_source_sha,base_digest:$base_digest,candidate_repository:$candidate_repository,candidate_digest:$candidate_digest,no_cache:true}' \ + > "${evidence_dir}/build.json" + + - name: Upload sanitized build evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: hbv4-${{ inputs.provider }}-build-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/hbv4-evidence + if-no-files-found: warn + retention-days: 90 + + cleanup_builder: + name: Cleanup ${{ inputs.provider }} builder + needs: [provision_builder, build_candidate] + if: ${{ always() }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + environment: azure-ci + permissions: + contents: read + id-token: write + steps: + - name: Checkout trusted cleanup action + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.control_sha }} + fetch-depth: 1 + persist-credentials: false + sparse-checkout: ci/azure/actions/cleanup-runner + sparse-checkout-cone-mode: false + + - name: Delete and verify builder VM and runner + if: ${{ always() }} + uses: ./ci/azure/actions/cleanup-runner + with: + provider: ${{ inputs.provider }} + runner_role: builder + provision_vm_name: ${{ needs.provision_builder.outputs.vm_name }} + provision_runner_id: ${{ needs.provision_builder.outputs.runner_id }} + azure_client_id: ${{ vars.AZURE_CLIENT_ID }} + azure_tenant_id: ${{ vars.AZURE_TENANT_ID }} + azure_subscription_id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + runner_resource_group: ${{ vars.AZURE_RUNNER_RESOURCE_GROUP }} + runner_app_client_id: ${{ vars.AZURE_RUNNER_APP_CLIENT_ID }} + runner_app_private_key: ${{ secrets.AZURE_RUNNER_APP_PRIVATE_KEY }} + + provision_tester: + name: Provision fresh ${{ inputs.provider }} tester + needs: [build_candidate, cleanup_builder] + if: ${{ needs.build_candidate.result == 'success' && needs.cleanup_builder.result == 'success' }} + runs-on: ubuntu-24.04 + timeout-minutes: 35 + environment: azure-ci + permissions: + contents: read + id-token: write + pull-requests: read + outputs: + runner_label: ${{ steps.provision.outputs.runner_label }} + vm_name: ${{ steps.provision.outputs.vm_name }} + runner_id: ${{ steps.provision.outputs.runner_id }} + provision_run_attempt: ${{ steps.provision.outputs.provision_run_attempt }} + steps: + - name: Checkout trusted runner control plane + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.control_sha }} + fetch-depth: 1 + persist-credentials: false + sparse-checkout: | + ci/azure + scripts/hbv4/validate-host.sh + scripts/hbv4/validate-hbv4-tpls + scripts/hbv4/parallel_hdf5_shared.c + sparse-checkout-cone-mode: false + + - name: Reauthorize live source before paid tester provisioning + env: + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ inputs.event_name }} + EVENT_REF: ${{ inputs.event_ref }} + PR_NUMBER: ${{ inputs.pr_number }} + SOURCE_SHA: ${{ inputs.source_sha }} + run: | + set -euo pipefail + case "${EVENT_NAME}" in + pull_request_target) + [[ "${PR_NUMBER}" =~ ^[1-9][0-9]*$ ]] + [[ "${EVENT_REF}" == "refs/pull/${PR_NUMBER}/head" ]] + live_pr="$(gh api -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" + jq -e \ + --arg repository "${GITHUB_REPOSITORY}" \ + --arg sha "${SOURCE_SHA}" \ + --argjson number "${PR_NUMBER}" ' + .number == $number and .state == "open" and + .head.repo.full_name == $repository and .head.sha == $sha and + any(.labels[]?; .name == "ci: build HBv4 TPL") + ' <<< "${live_pr}" >/dev/null || { + echo '::error::Live PR authorization changed before paid tester provisioning.' + exit 1 + } + ;; + workflow_dispatch) + [[ -z "${PR_NUMBER}" ]] + live_commit="$(gh api -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/commits/${SOURCE_SHA}")" + [[ "$(jq -r '.sha // empty' <<< "${live_commit}")" == "${SOURCE_SHA}" ]] || { + echo '::error::Manual source commit no longer resolves exactly.' + exit 1 + } + ;; + *) exit 2 ;; + esac + + - name: Provision tester JIT runner + id: provision + uses: ./ci/azure/actions/provision-hbv4-runner + with: + provider: ${{ inputs.provider }} + runner_role: tester + event_name: ${{ inputs.event_name }} + event_ref: ${{ inputs.event_ref }} + source_sha: ${{ inputs.source_sha }} + pr_number: ${{ inputs.pr_number }} + azure_client_id: ${{ vars.AZURE_CLIENT_ID }} + azure_tenant_id: ${{ vars.AZURE_TENANT_ID }} + azure_subscription_id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + foundation_resource_group: ${{ vars.AZURE_FOUNDATION_RESOURCE_GROUP }} + runner_resource_group: ${{ vars.AZURE_RUNNER_RESOURCE_GROUP }} + runner_vm_uami_client_id: ${{ vars.AZURE_RUNNER_VM_UAMI_CLIENT_ID }} + acr_name: ${{ vars.AZURE_ACR_NAME }} + # Match the reviewed earthflow-sim/GEOS procurement contract: the + # repository-level Default runner group has stable ID 1. + runner_group_id: '1' + runner_app_client_id: ${{ vars.AZURE_RUNNER_APP_CLIENT_ID }} + runner_app_private_key: ${{ secrets.AZURE_RUNNER_APP_PRIVATE_KEY }} + + test_candidate: + name: Validate ${{ inputs.provider }} digest offline + needs: [build_candidate, provision_tester] + if: ${{ needs.build_candidate.result == 'success' && needs.provision_tester.result == 'success' && needs.provision_tester.outputs.provision_run_attempt == format('{0}', github.run_attempt) }} + runs-on: ${{ needs.provision_tester.outputs.runner_label }} + timeout-minutes: 45 + environment: azure-ci + permissions: {} + outputs: + tested_digest: ${{ steps.test.outputs.tested_digest }} + evidence_sha256: ${{ steps.test.outputs.evidence_sha256 }} + validation_log_base64: ${{ steps.test.outputs.validation_log_base64 }} + trusted_validator_sha256: ${{ steps.test.outputs.trusted_validator_sha256 }} + trusted_source_sha256: ${{ steps.test.outputs.trusted_source_sha256 }} + steps: + # Deliberately no checkout, artifact action, OIDC permission, or token input. + - name: Pull exact digest and run network-disabled validation + id: test + env: + CANDIDATE_DIGEST: ${{ needs.build_candidate.outputs.candidate_digest }} + CANDIDATE_DIGEST_REFERENCE: ${{ needs.build_candidate.outputs.candidate_digest_reference }} + PROVIDER: ${{ inputs.provider }} + TMPDIR: /mnt/hbv4-local/container-tmp + run: | + set -euo pipefail + [[ "${CANDIDATE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] + [[ "${CANDIDATE_DIGEST_REFERENCE}" == *@"${CANDIDATE_DIGEST}" ]] + + trusted_validator='/opt/geos-ci/scripts/hbv4/validate-hbv4-tpls' + trusted_source='/opt/geos-ci/scripts/hbv4/parallel_hdf5_shared.c' + trusted_validator_sha256="$(sha256sum "${trusted_validator}" | awk '{print $1}')" + trusted_source_sha256="$(sha256sum "${trusted_source}" | awk '{print $1}')" + [[ "${trusted_validator_sha256}" =~ ^[0-9a-f]{64}$ ]] + [[ "${trusted_source_sha256}" =~ ^[0-9a-f]{64}$ ]] + evidence_file="${RUNNER_TEMP}/hbv4-container-validation.log" + { + printf 'trusted-validator-sha256=%s\n' "${trusted_validator_sha256}" + printf 'trusted-parallel-hdf5-source-sha256=%s\n' "${trusted_source_sha256}" + /opt/geos-ci/scripts/hbv4/validate-host.sh \ + --instance-type Standard_HB176rs_v4 \ + --compiler-flags 'target=zen4 -march=native -mtune=native' \ + --writable-path "${GITHUB_WORKSPACE}" + echo 'trusted-host-validation=success' + } 2>&1 | tee "${evidence_file}" + + docker pull "${CANDIDATE_DIGEST_REFERENCE}" + if ! docker image inspect \ + --format '{{range .RepoDigests}}{{println .}}{{end}}' \ + "${CANDIDATE_DIGEST_REFERENCE}" | \ + grep -Fx -- "${CANDIDATE_DIGEST_REFERENCE}" >/dev/null; then + echo '::error::Docker did not materialize the requested candidate digest.' + exit 1 + fi + candidate_registry="${CANDIDATE_DIGEST_REFERENCE%%/*}" + [[ "${candidate_registry}" =~ ^[a-z0-9]{5,50}\.azurecr\.io$ ]] + # The UAMI remains attached to the host by Azure, but the tester drops + # its cached Azure and Docker sessions before executing image content. + az account clear + docker logout "${candidate_registry}" >/dev/null + + # The mount root is intentionally root-owned. Create test scratch + # below the sticky RAID-backed container temporary directory instead. + validation_scratch="${TMPDIR}/container-validation-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + install -d -m 0777 "${validation_scratch}" + docker run --rm \ + --user 0:0 \ + --entrypoint /bin/bash \ + --workdir /work/hbv4-validation \ + --read-only \ + --network none \ + --security-opt no-new-privileges \ + --env BASH_ENV= \ + --env ENV= \ + --env LD_AUDIT= \ + --env LD_PRELOAD= \ + --shm-size 8g \ + --ulimit memlock=-1:-1 \ + --tmpfs /tmp:rw,nosuid,nodev,size=8g \ + --mount "type=bind,src=${validation_scratch},dst=/work/hbv4-validation" \ + --mount 'type=bind,src=/opt/geos-ci/scripts/hbv4,dst=/trusted-validation,readonly' \ + "${CANDIDATE_DIGEST_REFERENCE}" \ + --noprofile --norc /trusted-validation/validate-hbv4-tpls \ + --provider "${PROVIDER}" \ + --scratch /work/hbv4-validation 2>&1 | tee -a "${evidence_file}" + + evidence_size="$(wc -c < "${evidence_file}" | tr -d '[:space:]')" + if ! [[ "${evidence_size}" =~ ^[1-9][0-9]*$ ]] || (( evidence_size > 90000 )); then + echo '::error::Validation log is empty or exceeds the 90 KB transport limit.' + exit 1 + fi + if grep -Eiq '(password|passwd|secret|token|api[_-]?key|authorization|cookie)[[:space:]]*[:=]|https?://[^/@[:space:]]+:[^/@[:space:]]+@' \ + "${evidence_file}"; then + echo '::error::Validation log contains credential-like material and will not be transported.' + exit 1 + fi + + evidence_sha256="$(sha256sum "${evidence_file}" | awk '{print $1}')" + validation_log_base64="$(base64 -w 0 "${evidence_file}")" + [[ "${evidence_sha256}" =~ ^[0-9a-f]{64}$ ]] + { + echo "tested_digest=${CANDIDATE_DIGEST}" + echo "evidence_sha256=${evidence_sha256}" + echo "validation_log_base64=${validation_log_base64}" + echo "trusted_validator_sha256=${trusted_validator_sha256}" + echo "trusted_source_sha256=${trusted_source_sha256}" + } >> "${GITHUB_OUTPUT}" + + cleanup_tester: + name: Cleanup ${{ inputs.provider }} tester + needs: [provision_tester, test_candidate] + if: ${{ always() }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + environment: azure-ci + permissions: + contents: read + id-token: write + steps: + - name: Checkout trusted cleanup action + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.control_sha }} + fetch-depth: 1 + persist-credentials: false + sparse-checkout: ci/azure/actions/cleanup-runner + sparse-checkout-cone-mode: false + + - name: Delete and verify tester VM and runner + if: ${{ always() }} + uses: ./ci/azure/actions/cleanup-runner + with: + provider: ${{ inputs.provider }} + runner_role: tester + provision_vm_name: ${{ needs.provision_tester.outputs.vm_name }} + provision_runner_id: ${{ needs.provision_tester.outputs.runner_id }} + azure_client_id: ${{ vars.AZURE_CLIENT_ID }} + azure_tenant_id: ${{ vars.AZURE_TENANT_ID }} + azure_subscription_id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + runner_resource_group: ${{ vars.AZURE_RUNNER_RESOURCE_GROUP }} + runner_app_client_id: ${{ vars.AZURE_RUNNER_APP_CLIENT_ID }} + runner_app_private_key: ${{ secrets.AZURE_RUNNER_APP_PRIVATE_KEY }} + + finalize_candidate: + name: Promote exact ${{ inputs.provider }} digest + needs: + - provision_builder + - build_candidate + - cleanup_builder + - provision_tester + - test_candidate + - cleanup_tester + if: ${{ always() }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + environment: azure-ci + permissions: + contents: read + id-token: write + pull-requests: read + steps: + - name: Checkout trusted finalizer + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.control_sha }} + fetch-depth: 1 + persist-credentials: false + sparse-checkout: ci/azure-images/scripts/finalize-hbv4-candidate.sh + sparse-checkout-cone-mode: false + + - name: Reauthorize live source before immutable promotion + id: live_source + if: >- + ${{ needs.build_candidate.result == 'success' && + needs.cleanup_builder.result == 'success' && + needs.test_candidate.result == 'success' && + needs.cleanup_tester.result == 'success' }} + env: + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ inputs.event_name }} + PR_NUMBER: ${{ inputs.pr_number }} + SOURCE_SHA: ${{ inputs.source_sha }} + run: | + set -euo pipefail + if [[ "${EVENT_NAME}" == pull_request_target ]]; then + live_pr="$(gh api -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" + jq -e \ + --arg repository "${GITHUB_REPOSITORY}" \ + --arg sha "${SOURCE_SHA}" \ + --argjson number "${PR_NUMBER}" ' + .number == $number and .state == "open" and + .head.repo.full_name == $repository and .head.sha == $sha and + any(.labels[]?; .name == "ci: build HBv4 TPL") + ' <<< "${live_pr}" >/dev/null || { + echo '::error::PR head/label authorization changed; preserving candidate.' + exit 1 + } + else + live_commit="$(gh api -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/commits/${SOURCE_SHA}")" + [[ "$(jq -r '.sha // empty' <<< "${live_commit}")" == "${SOURCE_SHA}" ]] + fi + + - name: Login to Azure for exact-digest promotion + id: azure_login + if: ${{ steps.live_source.outcome == 'success' }} + uses: azure/login@f5d393ae46f8fde4be8b75f32e3fc50e654ad0ca # v3.0.1 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + - name: Promote exact tested digest in ACR + id: promote + if: ${{ steps.azure_login.outcome == 'success' }} + env: + ACR_NAME: ${{ vars.AZURE_ACR_NAME }} + CANDIDATE_REPOSITORY: ${{ needs.build_candidate.outputs.candidate_repository }} + CANDIDATE_DIGEST: ${{ needs.build_candidate.outputs.candidate_digest }} + TESTED_DIGEST: ${{ needs.test_candidate.outputs.tested_digest }} + PROVIDER: ${{ inputs.provider }} + SOURCE_SHA: ${{ inputs.source_sha }} + run: ci/azure-images/scripts/finalize-hbv4-candidate.sh + + - name: Write sanitized lifecycle evidence + if: ${{ always() }} + env: + PROVIDER: ${{ inputs.provider }} + SOURCE_SHA: ${{ inputs.source_sha }} + CONTROL_SHA: ${{ inputs.control_sha }} + ACR_NAME: ${{ vars.AZURE_ACR_NAME }} + BASE_DIGEST: ${{ needs.build_candidate.outputs.base_digest }} + CANDIDATE_REPOSITORY: ${{ needs.build_candidate.outputs.candidate_repository }} + CANDIDATE_DIGEST: ${{ needs.build_candidate.outputs.candidate_digest }} + TESTED_DIGEST: ${{ needs.test_candidate.outputs.tested_digest }} + TEST_EVIDENCE_SHA256: ${{ needs.test_candidate.outputs.evidence_sha256 }} + TEST_LOG_BASE64: ${{ needs.test_candidate.outputs.validation_log_base64 }} + TRUSTED_VALIDATOR_SHA256: ${{ needs.test_candidate.outputs.trusted_validator_sha256 }} + TRUSTED_SOURCE_SHA256: ${{ needs.test_candidate.outputs.trusted_source_sha256 }} + BUILD_RESULT: ${{ needs.build_candidate.result }} + CLEANUP_BUILDER_RESULT: ${{ needs.cleanup_builder.result }} + TEST_RESULT: ${{ needs.test_candidate.result }} + CLEANUP_TESTER_RESULT: ${{ needs.cleanup_tester.result }} + PROMOTE_OUTCOME: ${{ steps.promote.outcome }} + run: | + set -euo pipefail + evidence_dir="${RUNNER_TEMP}/hbv4-lifecycle-evidence" + install -d -m 0700 "${evidence_dir}" + [[ "${SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]] + [[ "${CONTROL_SHA}" =~ ^[0-9a-f]{40}$ ]] + [[ "${ACR_NAME}" =~ ^[a-z0-9]{5,50}$ ]] + if [[ -n "${TEST_LOG_BASE64}" ]]; then + printf '%s' "${TEST_LOG_BASE64}" | base64 --decode \ + > "${evidence_dir}/container-validation.log" + actual_test_sha256="$(sha256sum "${evidence_dir}/container-validation.log" | awk '{print $1}')" + if [[ "${actual_test_sha256}" != "${TEST_EVIDENCE_SHA256}" ]]; then + echo '::error::Transported tester evidence failed its SHA-256 check.' + exit 1 + fi + elif [[ "${TEST_RESULT}" == success ]]; then + echo '::error::Successful tester omitted its bounded validation log.' + exit 1 + fi + if [[ "${BUILD_RESULT}" == success ]]; then + [[ "${BASE_DIGEST}" =~ ^geosx/ubuntu@sha256:[0-9a-f]{64}$ ]] + fi + if [[ "${TEST_RESULT}" == success ]]; then + [[ "${TRUSTED_VALIDATOR_SHA256}" =~ ^[0-9a-f]{64}$ ]] + [[ "${TRUSTED_SOURCE_SHA256}" =~ ^[0-9a-f]{64}$ ]] + fi + final_acr_registry="${ACR_NAME}.azurecr.io" + final_repository="geosx/ubuntu24.04-gcc13-hbv4-${PROVIDER}" + final_tag="sha-${SOURCE_SHA}" + final_digest='' + if [[ "${PROMOTE_OUTCOME}" == success ]]; then + final_digest="${CANDIDATE_DIGEST}" + [[ "${final_digest}" =~ ^sha256:[0-9a-f]{64}$ ]] + final_reference="${final_acr_registry}/${final_repository}@${final_digest}" + { + printf '# Pull validated HBv4 %s image\n\n' "${PROVIDER}" + printf '```bash\n' + printf 'docker pull %s\n' "${final_reference}" + printf '```\n' + } > "${evidence_dir}/pull-instructions.md" + cat "${evidence_dir}/pull-instructions.md" >> "${GITHUB_STEP_SUMMARY}" + fi + jq -n \ + --arg provider "${PROVIDER}" \ + --arg tpl_source_sha "${SOURCE_SHA}" \ + --arg control_sha "${CONTROL_SHA}" \ + --arg base_digest "${BASE_DIGEST}" \ + --arg candidate_repository "${CANDIDATE_REPOSITORY}" \ + --arg candidate_digest "${CANDIDATE_DIGEST}" \ + --arg tested_digest "${TESTED_DIGEST}" \ + --arg test_evidence_sha256 "${TEST_EVIDENCE_SHA256}" \ + --arg trusted_validator_sha256 "${TRUSTED_VALIDATOR_SHA256}" \ + --arg trusted_source_sha256 "${TRUSTED_SOURCE_SHA256}" \ + --arg final_acr_registry "${final_acr_registry}" \ + --arg final_repository "${final_repository}" \ + --arg final_tag "${final_tag}" \ + --arg final_digest "${final_digest}" \ + --arg build_result "${BUILD_RESULT}" \ + --arg cleanup_builder_result "${CLEANUP_BUILDER_RESULT}" \ + --arg test_result "${TEST_RESULT}" \ + --arg cleanup_tester_result "${CLEANUP_TESTER_RESULT}" \ + --arg promote_outcome "${PROMOTE_OUTCOME}" \ + '{provider:$provider,tpl_source_sha:$tpl_source_sha,control_sha:$control_sha,base_digest:$base_digest,candidate_repository:$candidate_repository,candidate_digest:$candidate_digest,tested_digest:$tested_digest,test_evidence_sha256:$test_evidence_sha256,trusted_validator_sha256:$trusted_validator_sha256,trusted_parallel_hdf5_source_sha256:$trusted_source_sha256,final_acr:{registry:$final_acr_registry,repository:$final_repository,tag:$final_tag,digest:$final_digest},build_result:$build_result,cleanup_builder_result:$cleanup_builder_result,test_result:$test_result,cleanup_tester_result:$cleanup_tester_result,promote_outcome:$promote_outcome}' \ + > "${evidence_dir}/lifecycle.json" + + - name: Upload sanitized lifecycle evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: hbv4-${{ inputs.provider }}-lifecycle-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/hbv4-lifecycle-evidence + if-no-files-found: error + retention-days: 90 + + - name: Require complete successful lifecycle + if: ${{ always() }} + env: + PROVISION_BUILDER_RESULT: ${{ needs.provision_builder.result }} + BUILD_RESULT: ${{ needs.build_candidate.result }} + CLEANUP_BUILDER_RESULT: ${{ needs.cleanup_builder.result }} + PROVISION_TESTER_RESULT: ${{ needs.provision_tester.result }} + TEST_RESULT: ${{ needs.test_candidate.result }} + CLEANUP_TESTER_RESULT: ${{ needs.cleanup_tester.result }} + PROMOTE_OUTCOME: ${{ steps.promote.outcome }} + run: | + set -euo pipefail + failed=false + for result in \ + "${PROVISION_BUILDER_RESULT}" \ + "${BUILD_RESULT}" \ + "${CLEANUP_BUILDER_RESULT}" \ + "${PROVISION_TESTER_RESULT}" \ + "${TEST_RESULT}" \ + "${CLEANUP_TESTER_RESULT}"; do + if [[ "${result}" != success ]]; then failed=true; fi + done + if [[ "${PROMOTE_OUTCOME}" != success ]]; then failed=true; fi + [[ "${failed}" == false ]] diff --git a/.github/workflows/azure-janitor.yml b/.github/workflows/azure-janitor.yml new file mode 100644 index 00000000..3aa8da6e --- /dev/null +++ b/.github/workflows/azure-janitor.yml @@ -0,0 +1,76 @@ +name: Azure HBv4 janitor + +on: + schedule: + - cron: '23 */6 * * *' + workflow_dispatch: + +permissions: {} + +concurrency: + group: azure-hbv4-janitor-${{ github.repository }} + cancel-in-progress: false + +jobs: + janitor: + name: Reclaim expired HBv4 resources + runs-on: ubuntu-24.04 + timeout-minutes: 45 + environment: azure-ci + permissions: + actions: read + contents: read + id-token: write + steps: + - name: Require default-branch janitor control code + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + [[ "${GITHUB_REF}" == "refs/heads/${DEFAULT_BRANCH}" ]] || { + echo '::error::The Azure janitor may run only from the repository default branch.' + exit 1 + } + + - name: Checkout trusted janitor + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + fetch-depth: 1 + persist-credentials: false + sparse-checkout: ci/azure/scripts/janitor-hbv4.sh + sparse-checkout-cone-mode: false + + - name: Azure login + uses: azure/login@f5d393ae46f8fde4be8b75f32e3fc50e654ad0ca # v3.0.1 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + - name: Mint repository runner-administration token + id: runner_app_token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.AZURE_RUNNER_APP_CLIENT_ID }} + private-key: ${{ secrets.AZURE_RUNNER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} + + - name: Delete only expired exact-contract resources + env: + GH_TOKEN: ${{ steps.runner_app_token.outputs.token }} + ACTIONS_READ_TOKEN: ${{ github.token }} + AZURE_RUNNER_RESOURCE_GROUP: ${{ vars.AZURE_RUNNER_RESOURCE_GROUP }} + AZURE_ACR_NAME: ${{ vars.AZURE_ACR_NAME }} + EVIDENCE_FILE: ${{ runner.temp }}/hbv4-janitor-evidence/actions.jsonl + run: ci/azure/scripts/janitor-hbv4.sh + + - name: Upload sanitized janitor evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: hbv4-janitor-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/hbv4-janitor-evidence + if-no-files-found: warn + retention-days: 90 diff --git a/.github/workflows/docker_build_tpls_hbv4.yml b/.github/workflows/docker_build_tpls_hbv4.yml new file mode 100644 index 00000000..9ad51d16 --- /dev/null +++ b/.github/workflows/docker_build_tpls_hbv4.yml @@ -0,0 +1,220 @@ +name: Build HBv4 TPL images + +on: + workflow_dispatch: + inputs: + source_sha: + description: Full 40-character commit SHA from this repository to build + required: true + type: string + provider: + description: MPI provider image(s) to build + required: true + default: both + type: choice + options: + - both + - openmpi + - mpich + pull_request_target: + branches: + - earthflowAI + types: [labeled] + +permissions: {} + +concurrency: + group: hbv4-tpls-${{ github.repository }}-${{ github.event.pull_request.number || inputs.source_sha || github.run_id }} + cancel-in-progress: false + +jobs: + authorize: + name: Authorize exact source + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + pull-requests: read + outputs: + authorized: ${{ steps.source.outputs.authorized }} + source_sha: ${{ steps.source.outputs.source_sha }} + providers: ${{ steps.source.outputs.providers }} + event_name: ${{ steps.source.outputs.event_name }} + event_ref: ${{ steps.source.outputs.event_ref }} + pr_number: ${{ steps.source.outputs.pr_number }} + steps: + # Provisioning uses the workflow's trusted control revision rather than + # the requested TPL source. Exercise that same revision before any paid + # VM can be created so Azure response-schema regressions fail cheaply. + - name: Checkout trusted HBv4 control-plane tests + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + fetch-depth: 1 + persist-credentials: false + sparse-checkout: | + .github/workflows + ci/azure + docker/tpl-ubuntu.Dockerfile + docker/ubuntu-hbv4-openmpi-spack.yaml + docker/ubuntu-hbv4-mpich-spack.yaml + scripts/hbv4 + sparse-checkout-cone-mode: false + + - name: Validate trusted HBv4 control plane + run: | + set -euo pipefail + for test_script in ci/azure/tests/test-*.sh; do + bash "${test_script}" + done + python3 ci/azure/tests/test-render-runner-bootstrap.py + + - name: Validate trigger and live source identity + id: source + env: + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ github.event_name }} + EVENT_LABEL: ${{ github.event.label.name }} + EVENT_HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} + EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + EVENT_PR_NUMBER: ${{ github.event.pull_request.number }} + DISPATCH_SOURCE_SHA: ${{ inputs.source_sha }} + DISPATCH_PROVIDER: ${{ inputs.provider }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + + authorized=false + source_sha='' + providers='[]' + event_ref='' + pr_number='' + + case "${EVENT_NAME}" in + workflow_dispatch) + if [[ "${GITHUB_REF}" != "refs/heads/${DEFAULT_BRANCH}" ]]; then + echo '::error::Manual HBv4 control code must run from the repository default branch.' + exit 1 + fi + if ! [[ "${DISPATCH_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo '::error::source_sha must be exactly 40 lowercase hexadecimal characters.' + exit 1 + fi + case "${DISPATCH_PROVIDER}" in + both) providers='["openmpi","mpich"]' ;; + openmpi) providers='["openmpi"]' ;; + mpich) providers='["mpich"]' ;; + *) echo '::error::provider must be both, openmpi, or mpich.'; exit 1 ;; + esac + + commit_json="$(gh api \ + -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/commits/${DISPATCH_SOURCE_SHA}")" + live_sha="$(jq -r '.sha // empty' <<< "${commit_json}")" + if [[ "${live_sha}" != "${DISPATCH_SOURCE_SHA}" ]]; then + echo '::error::source_sha did not resolve to that exact repository commit.' + exit 1 + fi + authorized=true + source_sha="${DISPATCH_SOURCE_SHA}" + event_ref="${GITHUB_REF}" + ;; + + pull_request_target) + # Other labels are deliberate no-ops, not authorization failures. + if [[ "${EVENT_LABEL}" != 'ci: build HBv4 TPL' ]]; then + echo "Ignoring unrelated label '${EVENT_LABEL}'." + else + if ! [[ "${EVENT_PR_NUMBER}" =~ ^[1-9][0-9]*$ ]] || + ! [[ "${EVENT_HEAD_SHA}" =~ ^[0-9a-f]{40}$ ]] || + [[ "${EVENT_HEAD_REPOSITORY}" != "${GITHUB_REPOSITORY}" ]]; then + echo '::error::Only canonical same-repository pull requests can build HBv4 images.' + exit 1 + fi + + live_pr="$(gh api \ + -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/pulls/${EVENT_PR_NUMBER}")" + if ! jq -e \ + --arg repository "${GITHUB_REPOSITORY}" \ + --arg sha "${EVENT_HEAD_SHA}" \ + --argjson number "${EVENT_PR_NUMBER}" ' + .number == $number and + .state == "open" and + .head.repo.full_name == $repository and + .head.sha == $sha and + any(.labels[]?; .name == "ci: build HBv4 TPL") + ' <<< "${live_pr}" >/dev/null; then + echo '::error::The PR is closed, forked, stale, or no longer has the exact HBv4 build label.' + exit 1 + fi + + authorized=true + source_sha="${EVENT_HEAD_SHA}" + providers='["openmpi","mpich"]' + event_ref="refs/pull/${EVENT_PR_NUMBER}/head" + pr_number="${EVENT_PR_NUMBER}" + fi + ;; + + *) + echo "::error::Unsupported event ${EVENT_NAME}." + exit 1 + ;; + esac + + { + echo "authorized=${authorized}" + echo "source_sha=${source_sha}" + echo "providers=${providers}" + echo "event_name=${EVENT_NAME}" + echo "event_ref=${event_ref}" + echo "pr_number=${pr_number}" + } >> "${GITHUB_OUTPUT}" + + providers: + name: ${{ matrix.provider }} lifecycle + needs: authorize + if: ${{ needs.authorize.outputs.authorized == 'true' }} + # A reusable workflow cannot elevate above its caller. Grant only the union + # needed by its jobs; the called workflow narrows this per job. + permissions: + contents: read + id-token: write + pull-requests: read + strategy: + fail-fast: false + max-parallel: 2 + matrix: + provider: ${{ fromJSON(needs.authorize.outputs.providers) }} + uses: ./.github/workflows/_docker_build_tpls_hbv4_provider.yml + with: + provider: ${{ matrix.provider }} + source_sha: ${{ needs.authorize.outputs.source_sha }} + control_sha: ${{ github.sha }} + event_name: ${{ needs.authorize.outputs.event_name }} + event_ref: ${{ needs.authorize.outputs.event_ref }} + pr_number: ${{ needs.authorize.outputs.pr_number }} + secrets: inherit + + hbv4_tpl_result: + name: HBv4 TPL lifecycle result + needs: [authorize, providers] + if: ${{ always() }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: {} + steps: + - name: Require every authorized provider lifecycle + env: + AUTHORIZED: ${{ needs.authorize.outputs.authorized }} + AUTHORIZE_RESULT: ${{ needs.authorize.result }} + PROVIDERS_RESULT: ${{ needs.providers.result }} + run: | + set -euo pipefail + [[ "${AUTHORIZE_RESULT}" == 'success' ]] + if [[ "${AUTHORIZED}" == 'true' ]]; then + [[ "${PROVIDERS_RESULT}" == 'success' ]] + else + echo 'No HBv4 build was requested by this event.' + fi diff --git a/.uberenv_config.json b/.uberenv_config.json index b541fb7c..ebcf032b 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -6,8 +6,8 @@ "spack_configs_path": "scripts/spack_configs", "spack_packages_path": "scripts/spack_packages/packages", "spack_url": "https://github.com/spack/spack", -"spack_commit": "2e2169d5282d166f63e3ee4db8d4446c43cefa8a", -"spack_commit_note": "v1.1.1 (Jan 14th 2026)", -"spack_packages_commit": "3dd98680871078353a28ee508fa76c7554f918fa", -"spack_packages_note": "Feb 25th 2026" +"spack_commit": "3e19345b6e12f5ff1b874f4059622fc6a1fd804a", +"spack_commit_note": "v1.2.2 (Jul 20th 2026)", +"spack_packages_commit": "bd9b4838fbf9e4ca01ad7b6c6df633ddcc750d71", +"spack_packages_note": "Aug 5th 2026" } diff --git a/ci/azure-images/scripts/finalize-hbv4-candidate.sh b/ci/azure-images/scripts/finalize-hbv4-candidate.sh new file mode 100755 index 00000000..004b9098 --- /dev/null +++ b/ci/azure-images/scripts/finalize-hbv4-candidate.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash + +# Promote one tested run-scoped candidate. Every lookup is closed over the +# provider/run/source tuple and a non-future attempt; ambiguous or conflicting +# state exits before candidate deletion so the seven-day janitor window remains +# available and failed-only reruns can reuse their already-tested candidate. +set -euo pipefail + +: "${ACR_NAME:?}" +: "${CANDIDATE_REPOSITORY:?}" +: "${CANDIDATE_DIGEST:?}" +: "${TESTED_DIGEST:?}" +: "${PROVIDER:?}" +: "${SOURCE_SHA:?}" + +case "${PROVIDER}" in openmpi|mpich) ;; *) echo 'invalid provider' >&2; exit 2 ;; esac +[[ "${ACR_NAME}" =~ ^[a-z0-9]{5,50}$ ]] || exit 2 +[[ "${SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]] || exit 2 +[[ "${CANDIDATE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] || exit 2 +[[ "${TESTED_DIGEST}" == "${CANDIDATE_DIGEST}" ]] || { + echo 'tested digest does not equal candidate digest' >&2 + exit 1 +} +[[ "${GITHUB_RUN_ID}" =~ ^[1-9][0-9]*$ ]] || exit 2 +candidate_pattern="^geos/hbv4-tpl-candidates/${PROVIDER}/run-${GITHUB_RUN_ID}-attempt-([1-9][0-9]*)$" +[[ "${GITHUB_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] || exit 2 +[[ "${CANDIDATE_REPOSITORY}" =~ ${candidate_pattern} ]] || { + echo 'candidate repository is outside this run/provider namespace' >&2 + exit 1 +} +candidate_run_attempt="${BASH_REMATCH[1]}" +(( candidate_run_attempt <= GITHUB_RUN_ATTEMPT )) || { + echo 'candidate repository belongs to a future run attempt' >&2 + exit 1 +} + +destination_repository="geosx/ubuntu24.04-gcc13-hbv4-${PROVIDER}" +destination_tag="sha-${SOURCE_SHA}" +acr_login_server="${ACR_NAME}.azurecr.io" +acr_destination="${destination_repository}:${destination_tag}" + +repositories="$(az acr repository list --name "${ACR_NAME}" --output json --only-show-errors)" +jq -e 'type == "array" and all(.[]; type == "string")' <<< "${repositories}" >/dev/null || { + echo 'ACR repository inventory is malformed; preserving candidate' >&2 + exit 1 +} +candidate_count="$(jq -r --arg repo "${CANDIDATE_REPOSITORY}" '[.[] | select(. == $repo)] | length' <<< "${repositories}")" +[[ "${candidate_count}" == 1 ]] || { + echo 'ACR candidate repository is missing or ambiguous; preserving candidate' >&2 + exit 1 +} +live_candidate_digest="$(az acr repository show \ + --name "${ACR_NAME}" --image "${CANDIDATE_REPOSITORY}:candidate" \ + --query digest --output tsv --only-show-errors)" +[[ "${live_candidate_digest}" == "${CANDIDATE_DIGEST}" ]] || { + echo 'live ACR candidate digest changed; preserving candidate' >&2 + exit 1 +} + +existing_acr_digest='' +if jq -e --arg repo "${destination_repository}" 'any(.[]; . == $repo)' \ + <<< "${repositories}" >/dev/null; then + tags="$(az acr repository show-tags --name "${ACR_NAME}" \ + --repository "${destination_repository}" --detail --output json --only-show-errors)" + jq -e 'type == "array" and all(.[]; type == "object" and (.name | type) == "string")' \ + <<< "${tags}" >/dev/null || { + echo 'ACR destination tag inventory is malformed; preserving candidate' >&2 + exit 1 + } + matches="$(jq -c --arg tag "${destination_tag}" '[.[] | select(.name == $tag)]' <<< "${tags}")" + match_count="$(jq -r 'length' <<< "${matches}")" + (( match_count <= 1 )) || { echo 'duplicate ACR immutable tags' >&2; exit 1; } + if (( match_count == 1 )); then + existing_acr_digest="$(jq -r '.[0].digest // empty' <<< "${matches}")" + [[ "${existing_acr_digest}" =~ ^sha256:[0-9a-f]{64}$ ]] || { + echo 'ACR immutable tag has malformed digest' >&2 + exit 1 + } + fi +fi + +if [[ -n "${existing_acr_digest}" && "${existing_acr_digest}" != "${CANDIDATE_DIGEST}" ]]; then + echo "ACR immutable tag conflicts: ${existing_acr_digest}" >&2 + exit 1 +fi +if [[ -z "${existing_acr_digest}" ]]; then + az acr import \ + --name "${ACR_NAME}" \ + --source "${acr_login_server}/${CANDIDATE_REPOSITORY}@${CANDIDATE_DIGEST}" \ + --image "${acr_destination}" \ + --only-show-errors +fi +promoted_acr_digest="$(az acr repository show \ + --name "${ACR_NAME}" --image "${acr_destination}" \ + --query digest --output tsv --only-show-errors)" +[[ "${promoted_acr_digest}" == "${CANDIDATE_DIGEST}" ]] || { + echo 'ACR immutable promotion did not resolve to the tested digest' >&2 + exit 1 +} +az acr repository update --name "${ACR_NAME}" --image "${acr_destination}" \ + --write-enabled false --delete-enabled false --only-show-errors >/dev/null +az acr repository update --name "${ACR_NAME}" \ + --image "${destination_repository}@${CANDIDATE_DIGEST}" \ + --write-enabled false --delete-enabled false --only-show-errors >/dev/null +acr_tag_attributes="$(az acr repository show --name "${ACR_NAME}" \ + --image "${acr_destination}" --output json --only-show-errors)" +acr_manifest_attributes="$(az acr repository show --name "${ACR_NAME}" \ + --image "${destination_repository}@${CANDIDATE_DIGEST}" \ + --output json --only-show-errors)" +for acr_attributes in "${acr_tag_attributes}" "${acr_manifest_attributes}"; do + jq -e --arg digest "${CANDIDATE_DIGEST}" ' + .digest == $digest and + (.changeableAttributes | type) == "object" and + .changeableAttributes.writeEnabled == false and + .changeableAttributes.deleteEnabled == false + ' <<< "${acr_attributes}" >/dev/null || { + echo 'ACR promoted tag or manifest is not locked against overwrite/deletion' >&2 + exit 1 + } +done + +# Azure OIDC remains on this hosted finalizer. Pulling the promoted digest back +# through ACR verifies that the immutable destination is readable by digest and +# still resolves to the exact image tested on the fresh HBv4 runner. +az acr login --name "${ACR_NAME}" --only-show-errors >/dev/null +promoted_digest_reference="${acr_login_server}/${destination_repository}@${CANDIDATE_DIGEST}" +docker pull "${promoted_digest_reference}" >/dev/null +docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' \ + "${promoted_digest_reference}" | grep -Fx -- "${promoted_digest_reference}" >/dev/null + +# Only an unambiguous, locked, and readable ACR promotion removes the +# run-scoped candidate. +az acr repository delete --name "${ACR_NAME}" \ + --repository "${CANDIDATE_REPOSITORY}" --yes --only-show-errors +repositories="$(az acr repository list --name "${ACR_NAME}" --output json --only-show-errors)" +jq -e --arg repo "${CANDIDATE_REPOSITORY}" \ + 'type == "array" and all(.[]; type == "string") and all(.[]; . != $repo)' \ + <<< "${repositories}" >/dev/null || { + echo 'Candidate repository remains after successful promotion' >&2 + exit 1 +} + +echo "Promoted ${PROVIDER} ${CANDIDATE_DIGEST} to ${acr_login_server}/${acr_destination}." diff --git a/ci/azure/actions/cleanup-runner/action.yml b/ci/azure/actions/cleanup-runner/action.yml new file mode 100644 index 00000000..7795250f --- /dev/null +++ b/ci/azure/actions/cleanup-runner/action.yml @@ -0,0 +1,627 @@ +# TPL adaptation of the retrying, fail-closed earthflow-sim/GEOS teardown at +# db82dd7dfcbd60f9bf206e3efb0c97cb3568f5fd, extended with provider labels. +name: Cleanup HBv4 JIT runner +description: Resolve one exact provider/role runner, delete its VM, and verify its GitHub registration is absent. + +inputs: + provider: + required: true + runner_role: + required: true + provision_vm_name: + required: false + default: '' + provision_runner_id: + required: false + default: '' + azure_client_id: + required: true + azure_tenant_id: + required: true + azure_subscription_id: + required: true + runner_resource_group: + required: true + runner_app_client_id: + required: true + runner_app_private_key: + required: true + +runs: + using: composite + steps: + - name: Validate cleanup boundary + if: ${{ always() }} + shell: bash + env: + PROVIDER: ${{ inputs.provider }} + RUNNER_ROLE: ${{ inputs.runner_role }} + VM_NAME: ${{ inputs.provision_vm_name }} + RUNNER_ID: ${{ inputs.provision_runner_id }} + run: | + set -euo pipefail + case "${PROVIDER}" in openmpi|mpich) ;; *) exit 2 ;; esac + case "${RUNNER_ROLE}" in builder|tester) ;; *) exit 2 ;; esac + pattern="^geos-hbv4-${GITHUB_RUN_ID}-([1-9][0-9]*)-${PROVIDER}-${RUNNER_ROLE}$" + [[ -z "${VM_NAME}" || "${VM_NAME}" =~ ${pattern} ]] || { + echo '::error::Provisioned VM output is outside this run/provider/role boundary.' + exit 2 + } + [[ -z "${RUNNER_ID}" || "${RUNNER_ID}" =~ ^[1-9][0-9]*$ ]] || { + echo '::error::Provisioned runner ID is invalid.' + exit 2 + } + + - name: Azure login for teardown + if: ${{ always() }} + uses: azure/login@f5d393ae46f8fde4be8b75f32e3fc50e654ad0ca # v3.0.1 + with: + client-id: ${{ inputs.azure_client_id }} + tenant-id: ${{ inputs.azure_tenant_id }} + subscription-id: ${{ inputs.azure_subscription_id }} + + - name: Delete and verify exact Azure VM + if: ${{ always() }} + shell: bash + env: + PROVIDER: ${{ inputs.provider }} + RUNNER_ROLE: ${{ inputs.runner_role }} + PROVISION_VM_NAME: ${{ inputs.provision_vm_name }} + RUNNERS_RG: ${{ inputs.runner_resource_group }} + AZURE_CLIENT_ID: ${{ inputs.azure_client_id }} + AZURE_TENANT_ID: ${{ inputs.azure_tenant_id }} + AZURE_SUBSCRIPTION_ID: ${{ inputs.azure_subscription_id }} + run: | + set -euo pipefail + case "${PROVIDER}" in openmpi|mpich) ;; *) exit 2 ;; esac + case "${RUNNER_ROLE}" in builder|tester) ;; *) exit 2 ;; esac + pattern="^geos-hbv4-${GITHUB_RUN_ID}-([1-9][0-9]*)-${PROVIDER}-${RUNNER_ROLE}$" + + if ! az account show --only-show-errors >/dev/null 2>&1; then + [[ -n "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" && + -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]] || { + echo '::error::No Azure session or job OIDC request credential is available.' + exit 1 + } + oidc="$(curl --fail --silent --show-error \ + -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange" | jq -r '.value // empty')" + [[ -n "${oidc}" ]] || { echo '::error::OIDC endpoint returned no token.'; exit 1; } + az login --service-principal \ + --username "${AZURE_CLIENT_ID}" \ + --tenant "${AZURE_TENANT_ID}" \ + --federated-token "${oidc}" \ + --only-show-errors >/dev/null + az account set --subscription "${AZURE_SUBSCRIPTION_ID}" --only-show-errors + fi + + inventory="$(az vm list --resource-group "${RUNNERS_RG}" --output json --only-show-errors)" + jq -e 'type == "array"' <<< "${inventory}" >/dev/null || { + echo '::error::Azure VM inventory is malformed.' + exit 1 + } + + derived_vm_name="geos-hbv4-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${PROVIDER}-${RUNNER_ROLE}" + vm_name="${PROVISION_VM_NAME}" + if [[ -n "${vm_name}" ]]; then + [[ "${vm_name}" =~ ${pattern} ]] || exit 2 + provision_attempt="${BASH_REMATCH[1]}" + exact="$(jq -c --arg name "${vm_name}" '[.[] | select(.name == $name)]' <<< "${inventory}")" + count="$(jq -r 'length' <<< "${exact}")" + (( count <= 1 )) || { echo '::error::Duplicate exact Azure VM names.'; exit 1; } + if (( count == 1 )); then + if ! jq -e \ + --arg run "${GITHUB_RUN_ID}" \ + --arg attempt "${provision_attempt}" \ + --arg provider "${PROVIDER}" --arg role "${RUNNER_ROLE}" ' + .[0].tags.project == "GEOS" and .[0].tags.purpose == "hbv4-tpl-ci" and + (.[0].tags.githubRunId | tostring) == $run and + (.[0].tags.runAttempt | tostring) == $attempt and + .[0].tags.provider == $provider and .[0].tags.runnerRole == $role + ' <<< "${exact}" >/dev/null; then + echo '::error::Exact VM output exists but its immutable tags do not match.' + exit 1 + fi + else + vm_name='' + fi + fi + if [[ -z "${vm_name}" ]]; then + exact_current="$(jq -c --arg name "${derived_vm_name}" \ + '[.[] | select(.name == $name)]' <<< "${inventory}")" + current_count="$(jq -r 'length' <<< "${exact_current}")" + (( current_count <= 1 )) || { + echo '::error::Duplicate current-attempt Azure VM names.' + exit 1 + } + if (( current_count == 1 )); then + if ! jq -e \ + --arg run "${GITHUB_RUN_ID}" \ + --arg attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg provider "${PROVIDER}" \ + --arg role "${RUNNER_ROLE}" ' + .[0].tags.project == "GEOS" and + .[0].tags.purpose == "hbv4-tpl-ci" and + (.[0].tags.githubRunId | tostring) == $run and + (.[0].tags.runAttempt | tostring) == $attempt and + .[0].tags.provider == $provider and + .[0].tags.runnerRole == $role + ' <<< "${exact_current}" >/dev/null; then + echo '::error::Current-attempt VM name exists without the exact lifecycle tags.' + exit 1 + fi + vm_name="${derived_vm_name}" + fi + fi + if [[ -z "${vm_name}" ]]; then + matches="$(jq -c \ + --arg run "${GITHUB_RUN_ID}" --arg provider "${PROVIDER}" --arg role "${RUNNER_ROLE}" ' + [.[] | + select((.tags | type) == "object") | + select(.tags.project == "GEOS" and .tags.purpose == "hbv4-tpl-ci") | + select((.tags.githubRunId | tostring) == $run) | + select(.tags.provider == $provider and .tags.runnerRole == $role) | + .name] + ' <<< "${inventory}")" + jq -e 'type == "array" and all(.[]; type == "string")' <<< "${matches}" >/dev/null + count="$(jq -r 'length' <<< "${matches}")" + (( count <= 1 )) || { echo '::error::Ambiguous tag fallback found multiple VMs.'; exit 1; } + if (( count == 1 )); then vm_name="$(jq -r '.[0]' <<< "${matches}")"; fi + [[ -z "${vm_name}" || "${vm_name}" =~ ${pattern} ]] || { + echo '::error::Tag fallback resolved a noncanonical VM name.' + exit 1 + } + if [[ -n "${vm_name}" ]]; then + [[ "${vm_name}" =~ ${pattern} ]] + fallback_attempt="${BASH_REMATCH[1]}" + if ! jq -e --arg attempt "${fallback_attempt}" ' + (.[0].tags.runAttempt | tostring) == $attempt + ' <<< "$(jq -c --arg name "${vm_name}" '[.[] | select(.name == $name)]' <<< "${inventory}")" \ + >/dev/null; then + echo '::error::Tag fallback VM name and runAttempt tag disagree.' + exit 1 + fi + fi + fi + + nic_id='' + os_disk_id='' + target_attempt='' + if [[ -n "${vm_name}" ]]; then + [[ "${vm_name}" =~ ${pattern} ]] + target_attempt="${BASH_REMATCH[1]}" + attachments="$(az vm show --resource-group "${RUNNERS_RG}" --name "${vm_name}" \ + --query '{nic:networkProfile.networkInterfaces[0].id,osDisk:storageProfile.osDisk.managedDisk.id}' \ + --output json --only-show-errors)" + nic_id="$(jq -r '.nic // empty' <<< "${attachments}")" + os_disk_id="$(jq -r '.osDisk // empty' <<< "${attachments}")" + case "${nic_id,,}" in + */resourcegroups/"${RUNNERS_RG,,}"/providers/microsoft.network/networkinterfaces/*) ;; + *) echo '::error::VM attached NIC is outside the exact runner resource group/type.'; exit 1 ;; + esac + case "${os_disk_id,,}" in + */resourcegroups/"${RUNNERS_RG,,}"/providers/microsoft.compute/disks/*) ;; + *) echo '::error::VM attached OS disk is outside the exact runner resource group/type.'; exit 1 ;; + esac + is_expected_vm_not_found() { + local error_file="$1" + grep -Eq '\(ResourceNotFound\)|Code:[[:space:]]*ResourceNotFound' "${error_file}" && + grep -Fqi -- "${vm_name}" "${error_file}" && + grep -Fqi -- "${RUNNERS_RG}" "${error_file}" && + ! grep -Eiq 'SubscriptionNotFound|ResourceGroupNotFound' "${error_file}" + } + is_az_retryable() { + grep -Eiq \ + 'TooManyRequests|HTTP 429|throttl|timeout|temporar|InternalServerError|ServiceUnavailable|BadGateway|GatewayTimeout' \ + "$1" + } + delete_done=false + for delete_attempt in 1 2 3 4; do + delete_error="$(mktemp)" + if az vm delete --resource-group "${RUNNERS_RG}" --name "${vm_name}" \ + --yes --force-deletion true --only-show-errors 2>"${delete_error}"; then + rm -f "${delete_error}" + delete_done=true + break + fi + if is_expected_vm_not_found "${delete_error}"; then + rm -f "${delete_error}" + delete_done=true + break + fi + if (( delete_attempt < 4 )) && is_az_retryable "${delete_error}"; then + echo "az vm delete attempt ${delete_attempt} was transient; retrying." + sed -n '1,40p' "${delete_error}" >&2 + rm -f "${delete_error}" + sleep $(( delete_attempt * 15 )) + continue + fi + echo "::error::az vm delete failed for ${vm_name}; not accepting implicit success." + sed -n '1,40p' "${delete_error}" >&2 + rm -f "${delete_error}" + exit 1 + done + [[ "${delete_done}" == true ]] || { + echo "::error::VM deletion did not reach a known success state for ${vm_name}." + exit 1 + } + else + echo "No matching Azure VM exists for ${PROVIDER}/${RUNNER_ROLE}." + fi + + delete_exact_tagged_remnant() { + local child_id="$1" expected_type="$2" inventory matches count + [[ -n "${child_id}" ]] || return 0 + inventory="$(az resource list --resource-group "${RUNNERS_RG}" \ + --output json --only-show-errors)" + jq -e 'type == "array"' <<< "${inventory}" >/dev/null + matches="$(jq -c --arg id "${child_id}" \ + '[.[] | select((.id | ascii_downcase) == ($id | ascii_downcase))]' \ + <<< "${inventory}")" + count="$(jq -r 'length' <<< "${matches}")" + (( count <= 1 )) || { + echo '::error::Duplicate exact child resource IDs were returned.' + exit 1 + } + if (( count == 1 )); then + if ! jq -e \ + --arg type "${expected_type}" \ + --arg run "${GITHUB_RUN_ID}" \ + --arg attempt "${target_attempt}" \ + --arg provider "${PROVIDER}" \ + --arg role "${RUNNER_ROLE}" ' + (.[0].type | ascii_downcase) == ($type | ascii_downcase) and + .[0].tags.project == "GEOS" and + .[0].tags.purpose == "hbv4-tpl-ci" and + (.[0].tags.githubRunId | tostring) == $run and + (.[0].tags.runAttempt | tostring) == $attempt and + .[0].tags.provider == $provider and + .[0].tags.runnerRole == $role + ' <<< "${matches}" >/dev/null; then + echo '::error::Exact VM child remnant is untagged or outside this lifecycle.' + exit 1 + fi + az resource delete --ids "${child_id}" --only-show-errors + fi + for child_attempt in 1 2 3 4; do + inventory="$(az resource list --resource-group "${RUNNERS_RG}" \ + --output json --only-show-errors)" + count="$(jq -r --arg id "${child_id}" \ + '[.[] | select((.id | ascii_downcase) == ($id | ascii_downcase))] | length' \ + <<< "${inventory}")" + if (( count == 0 )); then return 0; fi + (( child_attempt < 4 )) || { + echo '::error::Exact VM child resource remains after deletion.' + exit 1 + } + sleep $(( child_attempt * 5 )) + done + } + delete_exact_tagged_remnant "${nic_id}" 'Microsoft.Network/networkInterfaces' + delete_exact_tagged_remnant "${os_disk_id}" 'Microsoft.Compute/disks' + + for attempt in 1 2 3 4; do + inventory="$(az vm list --resource-group "${RUNNERS_RG}" --output json --only-show-errors)" + jq -e 'type == "array"' <<< "${inventory}" >/dev/null + remaining_name=0 + if [[ -n "${vm_name}" ]]; then + remaining_name="$(jq -r --arg name "${vm_name}" \ + '[.[] | select(.name == $name)] | length' <<< "${inventory}")" + fi + remaining="$(jq -r --arg run "${GITHUB_RUN_ID}" --arg provider "${PROVIDER}" --arg role "${RUNNER_ROLE}" ' + [.[] | select((.tags | type) == "object") | + select((.tags.githubRunId | tostring) == $run and + .tags.provider == $provider and .tags.runnerRole == $role)] | length + ' <<< "${inventory}")" + if (( remaining_name == 0 && remaining == 0 )); then exit 0; fi + (( attempt < 4 )) || { echo '::error::Exact Azure VM remains after deletion.'; exit 1; } + sleep $(( attempt * 10 )) + done + + - name: Mint repository runner-administration token + id: runner_app_token + if: ${{ always() }} + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ inputs.runner_app_client_id }} + private-key: ${{ inputs.runner_app_private_key }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} + + - name: Delete and verify exact runner registration + if: ${{ always() }} + shell: bash + env: + GH_TOKEN: ${{ steps.runner_app_token.outputs.token }} + PROVIDER: ${{ inputs.provider }} + RUNNER_ROLE: ${{ inputs.runner_role }} + PROVISION_RUNNER_ID: ${{ inputs.provision_runner_id }} + run: | + set -euo pipefail + [[ -n "${GH_TOKEN}" ]] || { echo '::error::Runner administration token is absent.'; exit 1; } + case "${PROVIDER}" in openmpi|mpich) ;; *) exit 2 ;; esac + case "${RUNNER_ROLE}" in builder|tester) ;; *) exit 2 ;; esac + label_pattern="^geos-hbv4-${GITHUB_RUN_ID}-[1-9][0-9]*-${PROVIDER}-${RUNNER_ROLE}$" + expected_label="geos-hbv4-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${PROVIDER}-${RUNNER_ROLE}" + max_attempts=4 + + validate_inventory() { + jq -e ' + type == "array" and + all(.[]; + type == "object" and (.runners | type) == "array" and + all(.runners[]?; + (.id | type) == "number" and .id >= 1 and + .id == (.id | floor) and + (.name | type) == "string" and (.name | length) > 0 and + (.labels | type) == "array" and + all(.labels[]?; (.name | type) == "string") + ) + ) + ' >/dev/null + } + lifecycle_records() { + jq -c --arg regex "${label_pattern}" ' + [ + .[].runners[]? + | . as $runner + | [$runner.labels[]?.name | select(test($regex))] as $matching_labels + | select(($runner.name | test($regex)) or ($matching_labels | length) > 0) + | { + id: $runner.id, + name: $runner.name, + labels: [$runner.labels[].name], + matching_labels: $matching_labels, + canonical_name: ($runner.name | test($regex)) + } + ] + ' + } + + validate_lifecycle_records() { + jq -e ' + type == "array" and + all(.[]; + (.matching_labels | type) == "array" and + (.matching_labels | length) == 1 and + (.canonical_name == false or .name == .matching_labels[0]) + ) + ' >/dev/null + } + + is_gh_not_found() { + grep -Eiq 'HTTP 404|Not Found' "$1" + } + + is_gh_retryable() { + grep -Eiq \ + 'HTTP (408|429|5[0-9][0-9])|secondary rate limit|rate limit|timeout|temporar' \ + "$1" + } + + probe_runner_collection_access() { + local context="$1" response err_file attempt + for (( attempt = 1; attempt <= max_attempts; attempt++ )); do + err_file="$(mktemp)" + if response="$(gh api \ + -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/actions/runners?per_page=1" \ + 2>"${err_file}")"; then + rm -f "${err_file}" + if ! jq -e ' + type == "object" and + (.total_count | type) == "number" and + .total_count >= 0 and .total_count == (.total_count | floor) and + (.runners | type) == "array" + ' <<< "${response}" >/dev/null; then + echo "::error::Cannot classify ${context}: runner collection response is malformed." + return 1 + fi + return 0 + fi + if (( attempt < max_attempts )) && is_gh_retryable "${err_file}"; then + echo "Runner collection probe attempt ${attempt} for ${context} was transient; retrying." + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + sleep $(( attempt * 10 )) + continue + fi + echo "::error::Cannot classify ${context}: runner collection access failed." + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + return 1 + done + echo "::error::Cannot classify ${context}: runner collection access failed." + return 1 + } + + read_runner_inventory() { + local context="$1" response err_file attempt + for (( attempt = 1; attempt <= max_attempts; attempt++ )); do + err_file="$(mktemp)" + if response="$(gh api --paginate --slurp \ + -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/actions/runners?per_page=100" \ + 2>"${err_file}")"; then + rm -f "${err_file}" + if ! validate_inventory <<< "${response}"; then + echo "::error::${context} runner inventory is malformed." + return 1 + fi + RUNNER_PAGES="${response}" + return 0 + fi + if (( attempt < max_attempts )) && is_gh_retryable "${err_file}"; then + echo "Runner inventory attempt ${attempt} for ${context} was transient; retrying." + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + sleep $(( attempt * 10 )) + continue + fi + echo "::error::Unable to read ${context} runner inventory." + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + return 1 + done + echo "::error::Unable to read ${context} runner inventory." + return 1 + } + + RUNNER_PAGES='' + read_runner_inventory initial + pages="${RUNNER_PAGES}" + candidates="$(lifecycle_records <<< "${pages}")" + if ! validate_lifecycle_records <<< "${candidates}"; then + echo '::error::Runner inventory contains a malformed or inconsistent HBv4 lifecycle identity.' + exit 1 + fi + candidate_count="$(jq -r 'length' <<< "${candidates}")" + (( candidate_count <= 1 )) || { + echo "::error::Found multiple runner registrations for this run/provider/role: ${candidates}" + exit 1 + } + + runner_id="${PROVISION_RUNNER_ID}" + runner_name='' + runner_label='' + if [[ -n "${runner_id}" ]]; then + [[ "${runner_id}" =~ ^[1-9][0-9]*$ ]] || exit 2 + exact="$(jq -c --argjson id "${runner_id}" '[.[].runners[]? | select(.id == $id)]' <<< "${pages}")" + count="$(jq -r 'length' <<< "${exact}")" + (( count <= 1 )) || { echo '::error::Duplicate runner ID in inventory.'; exit 1; } + if (( count == 1 )); then + if (( candidate_count != 1 )) || + [[ "$(jq -r '.[0].id' <<< "${candidates}")" != "${runner_id}" ]]; then + echo '::error::Provisioned runner ID resolves outside this exact lifecycle.' + exit 1 + fi + runner_name="$(jq -r '.[0].name' <<< "${candidates}")" + runner_label="$(jq -r '.[0].matching_labels[0]' <<< "${candidates}")" + else + runner_id='' + fi + fi + if [[ -z "${runner_id}" ]]; then + # JIT registration can be created before the composite publishes any + # outputs. Resolve that failure mode by the exact unique label rather + # than assuming GitHub preserved a caller-provided runner name. + exact_current="$(jq -c --arg label "${expected_label}" ' + [.[].runners[]? | + . as $runner | + select(([.labels[]?.name | select(. == $label)] | length) == 1) | + {id:$runner.id,name:$runner.name,label:$label}] + ' <<< "${pages}")" + current_count="$(jq -r 'length' <<< "${exact_current}")" + (( current_count <= 1 )) || { + echo '::error::Duplicate current-attempt runner registrations.' + exit 1 + } + if (( current_count == 1 )); then + runner_id="$(jq -r '.[0].id' <<< "${exact_current}")" + runner_name="$(jq -r '.[0].name' <<< "${exact_current}")" + runner_label="${expected_label}" + fi + fi + if [[ -z "${runner_id}" ]]; then + if (( candidate_count == 1 )); then + runner_id="$(jq -r '.[0].id' <<< "${candidates}")" + runner_name="$(jq -r '.[0].name' <<< "${candidates}")" + runner_label="$(jq -r '.[0].matching_labels[0]' <<< "${candidates}")" + fi + fi + + if [[ -n "${runner_id}" ]]; then + [[ "${runner_id}" =~ ^[1-9][0-9]*$ ]] || exit 2 + [[ "${runner_label}" =~ ${label_pattern} ]] || { + echo '::error::Resolved runner is missing its exact lifecycle label.' + exit 1 + } + delete_done=false + for (( attempt = 1; attempt <= max_attempts; attempt++ )); do + err_file="$(mktemp)" + if gh api --method DELETE \ + -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/actions/runners/${runner_id}" \ + >/dev/null 2>"${err_file}"; then + rm -f "${err_file}" + delete_done=true + break + fi + if is_gh_not_found "${err_file}"; then + rm -f "${err_file}" + probe_runner_collection_access "runner DELETE 404 for ${runner_id}" || exit 1 + delete_done=true + break + fi + if (( attempt < max_attempts )) && is_gh_retryable "${err_file}"; then + echo "Runner DELETE attempt ${attempt} was transient; retrying." + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + sleep $(( attempt * 10 )) + continue + fi + echo "::error::GitHub runner DELETE failed for ${runner_id}." + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + exit 1 + done + [[ "${delete_done}" == true ]] || { + echo "::error::Runner deletion did not reach a known success state for ${runner_id}." + exit 1 + } + else + echo "No matching GitHub runner registration exists for ${PROVIDER}/${RUNNER_ROLE}." + fi + + if [[ -n "${runner_id}" ]]; then + for (( attempt = 1; attempt <= max_attempts; attempt++ )); do + err_file="$(mktemp)" + if response="$(gh api \ + -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/actions/runners/${runner_id}" \ + 2>"${err_file}")"; then + rm -f "${err_file}" + if ! jq -e --argjson id "${runner_id}" ' + type == "object" and .id == $id + ' <<< "${response}" >/dev/null; then + echo "::error::Runner GET returned a malformed identity for ${runner_id}." + exit 1 + fi + if (( attempt < max_attempts )); then + echo "Runner ${runner_id} remains visible after verification attempt ${attempt}; retrying." + sleep $(( attempt * 10 )) + continue + fi + echo "::error::Runner registration ${runner_id} still exists after deletion." + exit 1 + fi + if is_gh_not_found "${err_file}"; then + rm -f "${err_file}" + probe_runner_collection_access "runner verification 404 for ${runner_id}" || exit 1 + break + fi + if (( attempt < max_attempts )) && is_gh_retryable "${err_file}"; then + echo "Runner GET verification attempt ${attempt} was transient; retrying." + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + sleep $(( attempt * 10 )) + continue + fi + echo "::error::Unable to verify runner registration ${runner_id} was deleted." + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + exit 1 + done + fi + + read_runner_inventory verification + remaining="$(lifecycle_records <<< "${RUNNER_PAGES}")" + if ! validate_lifecycle_records <<< "${remaining}"; then + echo '::error::Runner verification found a malformed HBv4 lifecycle identity.' + exit 1 + fi + if (( $(jq -r 'length' <<< "${remaining}") != 0 )); then + echo "::error::HBv4 lifecycle registration remains after deletion: ${remaining}" + exit 1 + fi + echo "Runner registration ${runner_id:-none}${runner_name:+ (${runner_name})} is absent." diff --git a/ci/azure/actions/provision-hbv4-runner/action.yml b/ci/azure/actions/provision-hbv4-runner/action.yml new file mode 100644 index 00000000..02afeeb7 --- /dev/null +++ b/ci/azure/actions/provision-hbv4-runner/action.yml @@ -0,0 +1,562 @@ +# TPL adaptation of the reviewed earthflow-sim/GEOS HBv4 lifecycle at +# db82dd7dfcbd60f9bf206e3efb0c97cb3568f5fd. Provider-specific image inputs, +# tags, and trusted validators are the only intended behavioral extensions. +name: Provision HBv4 JIT runner +description: Provision one fixed, private, six-hour HBv4 runner and wait for its exact JIT registration. + +inputs: + provider: + description: openmpi or mpich lifecycle owner. + required: true + runner_role: + description: builder or tester. + required: true + event_name: + description: workflow_dispatch or pull_request_target. + required: true + event_ref: + description: Canonical source event ref. + required: true + source_sha: + description: Exact same-repository source commit. + required: true + pr_number: + description: Pull request number, empty for workflow_dispatch. + required: false + default: '' + azure_client_id: + required: true + azure_tenant_id: + required: true + azure_subscription_id: + required: true + foundation_resource_group: + required: true + runner_resource_group: + required: true + runner_vm_uami_client_id: + description: Client ID matching the existing id-geos-runner-vm workload UAMI. + required: true + acr_name: + required: true + runner_group_id: + required: true + runner_app_client_id: + required: true + runner_app_private_key: + required: true + +outputs: + runner_label: + value: ${{ steps.identity.outputs.runner_label }} + vm_name: + value: ${{ steps.create_vm.outputs.vm_name }} + runner_id: + # Capture the registration at JIT creation, before any Azure operation can + # fail. Cleanup can therefore delete a pre-VM registration by exact ID. + value: ${{ steps.jit.outputs.runner_id }} + provision_run_attempt: + description: GitHub run attempt incorporated into the provisioned identity. + value: ${{ steps.identity.outputs.provision_run_attempt }} + +runs: + using: composite + steps: + - name: Validate lifecycle identity and fixed HBv4 contract + id: identity + shell: bash + env: + HBV4_PROVIDER: ${{ inputs.provider }} + HBV4_ROLE: ${{ inputs.runner_role }} + HBV4_EVENT_NAME: ${{ inputs.event_name }} + HBV4_EVENT_REF: ${{ inputs.event_ref }} + HBV4_SOURCE_SHA: ${{ inputs.source_sha }} + HBV4_PR_NUMBER: ${{ inputs.pr_number }} + HBV4_ACR_NAME: ${{ inputs.acr_name }} + HBV4_RUNNER_GROUP_ID: ${{ inputs.runner_group_id }} + run: | + set -euo pipefail + + case "${HBV4_PROVIDER}" in openmpi|mpich) ;; *) echo '::error::Invalid provider.'; exit 2 ;; esac + case "${HBV4_ROLE}" in builder|tester) ;; *) echo '::error::Invalid runner role.'; exit 2 ;; esac + case "${HBV4_EVENT_NAME}" in workflow_dispatch|pull_request_target) ;; + *) echo '::error::This event cannot provision a paid runner.'; exit 2 ;; + esac + [[ "${HBV4_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]] || { echo '::error::Invalid source SHA.'; exit 2; } + [[ "${HBV4_ACR_NAME}" =~ ^[a-z0-9]{5,50}$ ]] || { echo '::error::Invalid ACR name.'; exit 2; } + [[ "${HBV4_RUNNER_GROUP_ID}" =~ ^[1-9][0-9]*$ ]] || { echo '::error::Invalid runner group ID.'; exit 2; } + [[ "${GITHUB_RUN_ID}" =~ ^[1-9][0-9]*$ ]] || { echo '::error::Invalid GitHub run ID.'; exit 2; } + [[ "${GITHUB_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] || { echo '::error::Invalid GitHub run attempt.'; exit 2; } + if [[ "${HBV4_EVENT_NAME}" == pull_request_target ]]; then + [[ "${HBV4_PR_NUMBER}" =~ ^[1-9][0-9]*$ ]] || { echo '::error::Invalid PR number.'; exit 2; } + [[ "${HBV4_EVENT_REF}" == "refs/pull/${HBV4_PR_NUMBER}/head" ]] || { echo '::error::Invalid PR source ref.'; exit 2; } + elif [[ -n "${HBV4_PR_NUMBER}" ]]; then + echo '::error::workflow_dispatch must not carry a PR number.' + exit 2 + fi + + instance_key="${HBV4_PROVIDER}-${HBV4_ROLE}" + [[ "${instance_key}" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || { + echo '::error::Runner instance key is not a canonical lowercase ASCII slug.' + exit 1 + } + + for helper in \ + ci/azure/scripts/runner-names.sh \ + ci/azure/scripts/render-runner-bootstrap.py \ + ci/azure/scripts/runner-bootstrap.sh \ + ci/azure/scripts/validate-created-hbv4-vm.sh \ + ci/azure/scripts/setup-hbv4-local-nvme.sh \ + ci/azure/scripts/probe-runner-bootstrap.sh \ + scripts/hbv4/validate-host.sh \ + scripts/hbv4/validate-hbv4-tpls \ + scripts/hbv4/parallel_hdf5_shared.c; do + [[ -f "${GITHUB_WORKSPACE}/${helper}" ]] || { + echo "::error::Trusted runner helper is missing: ${helper}" + exit 1 + } + done + + identity_output="$(bash "${GITHUB_WORKSPACE}/ci/azure/scripts/runner-names.sh" \ + "${GITHUB_RUN_ID}" "${GITHUB_RUN_ATTEMPT}" "${instance_key}")" + for field in runner_name runner_label vm_name log_blob_prefix; do + field_count="$(awk -F= -v field="${field}" '$1 == field { count += 1 } END { print count + 0 }' \ + <<< "${identity_output}")" + field_value="$(awk -F= -v field="${field}" '$1 == field { sub(/^[^=]*=/, ""); print }' \ + <<< "${identity_output}")" + if [[ "${field_count}" != 1 || -z "${field_value}" ]]; then + echo "::error::runner-names.sh did not emit exactly one nonempty ${field}." + exit 1 + fi + done + + expected_name="geos-hbv4-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${instance_key}" + (( ${#expected_name} <= 64 )) || { echo '::error::Runner identity exceeds Azure limit.'; exit 1; } + actual_runner_name="$(awk -F= '$1 == "runner_name" { sub(/^[^=]*=/, ""); print }' <<< "${identity_output}")" + actual_runner_label="$(awk -F= '$1 == "runner_label" { sub(/^[^=]*=/, ""); print }' <<< "${identity_output}")" + actual_vm_name="$(awk -F= '$1 == "vm_name" { sub(/^[^=]*=/, ""); print }' <<< "${identity_output}")" + if [[ "${actual_runner_name}" != "${expected_name}" || + "${actual_runner_label}" != "${expected_name}" || + "${actual_vm_name}" != "${expected_name}" ]]; then + echo "::error::runner-names.sh did not preserve the reconstructable lifecycle identity." + exit 1 + fi + { + printf '%s\n' "${identity_output}" + echo "runner_instance_key=${instance_key}" + echo "provision_run_attempt=${GITHUB_RUN_ATTEMPT}" + } >> "${GITHUB_OUTPUT}" + + - name: Azure login for control-plane provisioning + uses: azure/login@f5d393ae46f8fde4be8b75f32e3fc50e654ad0ca # v3.0.1 + with: + client-id: ${{ inputs.azure_client_id }} + tenant-id: ${{ inputs.azure_tenant_id }} + subscription-id: ${{ inputs.azure_subscription_id }} + + - name: Mint repository runner-administration token + id: runner_app_token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ inputs.runner_app_client_id }} + private-key: ${{ inputs.runner_app_private_key }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} + + - name: Resolve private subnet and pull-only workload identity + id: azure_ids + shell: bash + env: + HBV4_FOUNDATION_RG: ${{ inputs.foundation_resource_group }} + HBV4_EXPECTED_UAMI_CLIENT_ID: ${{ inputs.runner_vm_uami_client_id }} + HBV4_ACR_NAME: ${{ inputs.acr_name }} + run: | + set -euo pipefail + subnet_id="$(az network vnet subnet show \ + --resource-group "${HBV4_FOUNDATION_RG}" \ + --vnet-name vnet-geos-ci \ + --name snet-runners \ + --query id --output tsv --only-show-errors)" + uami_json="$(az identity show \ + --resource-group "${HBV4_FOUNDATION_RG}" \ + --name id-geos-runner-vm \ + --output json --only-show-errors)" + uami_id="$(jq -r '.id // empty' <<< "${uami_json}")" + uami_client_id="$(jq -r '.clientId // empty' <<< "${uami_json}")" + uami_principal_id="$(jq -r '.principalId // empty' <<< "${uami_json}")" + if [[ -z "${subnet_id}" || -z "${uami_id}" || + -z "${uami_principal_id}" || + "${uami_client_id,,}" != "${HBV4_EXPECTED_UAMI_CLIENT_ID,,}" ]]; then + echo '::error::Private subnet or fixed workload UAMI does not match configuration.' + exit 1 + fi + + acr_json="$(az acr show --name "${HBV4_ACR_NAME}" \ + --query '{id:id,roleAssignmentMode:roleAssignmentMode}' \ + --output json --only-show-errors)" + acr_id="$(jq -r '.id // empty' <<< "${acr_json}")" + acr_role_assignment_mode="$(jq -r '.roleAssignmentMode // empty' <<< "${acr_json}")" + if [[ -z "${acr_id}" || "${acr_role_assignment_mode}" != 'LegacyRegistryPermissions' ]]; then + echo '::error::Shared ACR must use its existing legacy RBAC permission mode.' + exit 1 + fi + # Do not inspect role assignments here: the narrowly scoped workflow + # identity intentionally lacks Microsoft.Authorization/roleAssignments/read. + # The guest bootstrap proves the fixed UAMI's AcrPull access with an + # operational ACR login before the JIT runner is brought online. + { + echo "subnet_id=${subnet_id}" + echo "uami_id=${uami_id}" + echo "uami_client_id=${uami_client_id}" + } >> "${GITHUB_OUTPUT}" + + - name: Refuse overlapping provider VM or registration + shell: bash + env: + GH_TOKEN: ${{ steps.runner_app_token.outputs.token }} + HBV4_RUNNERS_RG: ${{ inputs.runner_resource_group }} + HBV4_PROVIDER: ${{ inputs.provider }} + HBV4_EXPECTED_LABEL: ${{ steps.identity.outputs.runner_label }} + run: | + set -euo pipefail + vm_inventory="$(az vm list --resource-group "${HBV4_RUNNERS_RG}" --output json --only-show-errors)" + if ! jq -e 'type == "array"' <<< "${vm_inventory}" >/dev/null; then + echo '::error::Azure VM inventory is malformed.' + exit 1 + fi + overlap_count="$(jq -r --arg provider "${HBV4_PROVIDER}" '[.[] | + select((.tags | type) == "object") | + select(.tags.project == "GEOS" and .tags.purpose == "hbv4-tpl-ci") | + select(.tags.provider == $provider)] | length' <<< "${vm_inventory}")" + if (( overlap_count != 0 )); then + echo "::error::Provider ${HBV4_PROVIDER} already owns an HBv4 VM; cleanup or janitor must reclaim it first." + exit 1 + fi + + runner_pages="$(gh api --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/actions/runners?per_page=100")" + if ! jq -e ' + type == "array" and + all(.[]; + (.runners | type) == "array" and + all(.runners[]?; + (.id | type) == "number" and .id >= 1 and .id == (.id | floor) and + (.name | type) == "string" and (.name | length) > 0 and + (.labels | type) == "array" and + all(.labels[]?; (.name | type) == "string") + ) + ) + ' \ + <<< "${runner_pages}" >/dev/null; then + echo '::error::GitHub runner inventory is malformed.' + exit 1 + fi + provider_label_pattern="^geos-hbv4-[1-9][0-9]*-[1-9][0-9]*-${HBV4_PROVIDER}-(builder|tester)$" + if jq -e --arg expected "${HBV4_EXPECTED_LABEL}" --arg regex "${provider_label_pattern}" ' + any(.[].runners[]?; + (.name | test($regex)) or + any(.labels[]?; .name == $expected or (.name | test($regex)))) + ' <<< "${runner_pages}" >/dev/null; then + echo "::error::Provider ${HBV4_PROVIDER} already owns an HBv4 runner registration; cleanup or janitor must reclaim it first." + exit 1 + fi + + - name: Generate exact-label single-use JIT configuration + id: jit + shell: bash + env: + GH_TOKEN: ${{ steps.runner_app_token.outputs.token }} + HBV4_JIT_NAME: ${{ steps.identity.outputs.runner_name }} + HBV4_JIT_LABEL: ${{ steps.identity.outputs.runner_label }} + HBV4_RUNNER_GROUP_ID: ${{ inputs.runner_group_id }} + run: | + set -euo pipefail + response="$(gh api --method POST \ + -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/actions/runners/generate-jitconfig" \ + -f "name=${HBV4_JIT_NAME}" \ + -F "runner_group_id=${HBV4_RUNNER_GROUP_ID}" \ + -f "labels[]=${HBV4_JIT_LABEL}" \ + -f 'work_folder=_work')" + encoded="$(jq -r '.encoded_jit_config // empty' <<< "${response}")" + runner_id="$(jq -r '.runner.id // empty' <<< "${response}")" + if [[ -z "${encoded}" ]] || ! [[ "${runner_id}" =~ ^[1-9][0-9]*$ ]] || + ! jq -e \ + --arg name "${HBV4_JIT_NAME}" \ + --arg label "${HBV4_JIT_LABEL}" ' + (.runner.id | type) == "number" and + .runner.id >= 1 and .runner.id == (.runner.id | floor) and + .runner.name == $name and + ([.runner.labels[]? | select(.name == $label)] | length) == 1 + ' <<< "${response}" >/dev/null; then + echo '::error::GitHub returned an incomplete or mismatched JIT runner configuration.' + exit 1 + fi + echo "::add-mask::${encoded}" + { + echo "encoded=${encoded}" + echo "runner_id=${runner_id}" + } >> "${GITHUB_OUTPUT}" + + - name: Render reviewed custom data + id: bootstrap + shell: bash + env: + RUNNER_ENCODED_JITCONFIG: ${{ steps.jit.outputs.encoded }} + BOOTSTRAP_IDENTITY: ${{ steps.identity.outputs.vm_name }} + ACR_NAME: ${{ inputs.acr_name }} + RUNNER_VM_UAMI_CLIENT_ID: ${{ steps.azure_ids.outputs.uami_client_id }} + RUNNER_STORAGE_PROFILE: hbv4-nvme-raid0 + run: | + set -euo pipefail + storage_helper="${GITHUB_WORKSPACE}/ci/azure/scripts/setup-hbv4-local-nvme.sh" + host_validator="${GITHUB_WORKSPACE}/scripts/hbv4/validate-host.sh" + container_validator="${GITHUB_WORKSPACE}/scripts/hbv4/validate-hbv4-tpls" + parallel_hdf5_source="${GITHUB_WORKSPACE}/scripts/hbv4/parallel_hdf5_shared.c" + STORAGE_HELPER_SHA256="$(sha256sum "${storage_helper}" | awk '{print $1}')" + STORAGE_HELPER_GZIP_BASE64="$(gzip -n -9 -c "${storage_helper}" | base64 -w 0)" + HOST_VALIDATOR_SHA256="$(sha256sum "${host_validator}" | awk '{print $1}')" + HOST_VALIDATOR_GZIP_BASE64="$(gzip -n -9 -c "${host_validator}" | base64 -w 0)" + CONTAINER_VALIDATOR_SHA256="$(sha256sum "${container_validator}" | awk '{print $1}')" + CONTAINER_VALIDATOR_GZIP_BASE64="$(gzip -n -9 -c "${container_validator}" | base64 -w 0)" + PARALLEL_HDF5_SOURCE_SHA256="$(sha256sum "${parallel_hdf5_source}" | awk '{print $1}')" + PARALLEL_HDF5_SOURCE_GZIP_BASE64="$(gzip -n -9 -c "${parallel_hdf5_source}" | base64 -w 0)" + export STORAGE_HELPER_SHA256 STORAGE_HELPER_GZIP_BASE64 + export HOST_VALIDATOR_SHA256 HOST_VALIDATOR_GZIP_BASE64 + export CONTAINER_VALIDATOR_SHA256 CONTAINER_VALIDATOR_GZIP_BASE64 + export PARALLEL_HDF5_SOURCE_SHA256 PARALLEL_HDF5_SOURCE_GZIP_BASE64 + export RUNNER_VERSION='2.335.1' + export RUNNER_TARBALL_SHA256='4ef2f25285f0ae4477f1fe1e346db76d2f3ebf03824e2ddd1973a2819bf6c8cf' + + output="${RUNNER_TEMP}/${BOOTSTRAP_IDENTITY}-bootstrap.sh" + python3 "${GITHUB_WORKSPACE}/ci/azure/scripts/render-runner-bootstrap.py" \ + "${GITHUB_WORKSPACE}/ci/azure/scripts/runner-bootstrap.sh" "${output}" + chmod 0600 "${output}" + echo "custom_data=${output}" >> "${GITHUB_OUTPUT}" + + - name: Create fixed paid VM from the authorized source + id: create_vm + shell: bash + env: + HBV4_EVENT_NAME: ${{ inputs.event_name }} + HBV4_EVENT_REF: ${{ inputs.event_ref }} + HBV4_SOURCE_SHA: ${{ inputs.source_sha }} + HBV4_PR_NUMBER: ${{ inputs.pr_number }} + HBV4_PROVIDER: ${{ inputs.provider }} + HBV4_ROLE: ${{ inputs.runner_role }} + HBV4_VM_NAME: ${{ steps.identity.outputs.vm_name }} + HBV4_JIT_LABEL: ${{ steps.identity.outputs.runner_label }} + HBV4_INSTANCE_KEY: ${{ steps.identity.outputs.runner_instance_key }} + HBV4_LOG_BLOB_PREFIX: ${{ steps.identity.outputs.log_blob_prefix }} + HBV4_RUNNERS_RG: ${{ inputs.runner_resource_group }} + HBV4_SUBNET_ID: ${{ steps.azure_ids.outputs.subnet_id }} + HBV4_UAMI_ID: ${{ steps.azure_ids.outputs.uami_id }} + HBV4_CUSTOM_DATA: ${{ steps.bootstrap.outputs.custom_data }} + run: | + set -euo pipefail + # As in the GEOS runtime-image lifecycle, source authorization belongs + # to the calling workflow and its read-only GITHUB_TOKEN. This composite + # keeps the runner-administration App token limited to runner APIs and + # revalidates only the prepared event contract at the paid boundary. + case "${HBV4_EVENT_NAME}" in + pull_request_target) + [[ "${HBV4_PR_NUMBER}" =~ ^[1-9][0-9]*$ ]] + [[ "${HBV4_EVENT_REF}" == "refs/pull/${HBV4_PR_NUMBER}/head" ]] + [[ "${HBV4_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]] + ;; + workflow_dispatch) + [[ -z "${HBV4_PR_NUMBER}" ]] + [[ "${HBV4_EVENT_REF}" == refs/heads/* ]] + [[ "${HBV4_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]] + ;; + *) exit 2 ;; + esac + + readonly VM_SIZE='Standard_HB176rs_v4' + readonly VM_IMAGE='microsoft-dsvm:ubuntu-hpc:2404:24.04.2026070201' + readonly LOCATION='southcentralus' + created_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + az vm create \ + --resource-group "${HBV4_RUNNERS_RG}" \ + --name "${HBV4_VM_NAME}" \ + --location "${LOCATION}" \ + --image "${VM_IMAGE}" \ + --size "${VM_SIZE}" \ + --subnet "${HBV4_SUBNET_ID}" \ + --public-ip-address '' \ + --nsg '' \ + --accelerated-networking true \ + --assign-identity "${HBV4_UAMI_ID}" \ + --admin-username azureci \ + --generate-ssh-keys \ + --os-disk-size-gb 128 \ + --os-disk-delete-option Delete \ + --nic-delete-option Delete \ + --custom-data "${HBV4_CUSTOM_DATA}" \ + --tags \ + project=GEOS \ + purpose=hbv4-tpl-ci \ + provider="${HBV4_PROVIDER}" \ + runnerRole="${HBV4_ROLE}" \ + githubRunId="${GITHUB_RUN_ID}" \ + runAttempt="${GITHUB_RUN_ATTEMPT}" \ + runnerInstanceKey="${HBV4_INSTANCE_KEY}" \ + runnerLabel="${HBV4_JIT_LABEL}" \ + logBlobPrefix="${HBV4_LOG_BLOB_PREFIX}" \ + sourceSha="${HBV4_SOURCE_SHA}" \ + createdAtUtc="${created_at}" \ + ttlHours=6 \ + --only-show-errors >/dev/null + + # Validate the unprojected Azure response. The shared helper and its + # realistic fixture therefore exercise the same case-sensitive Compute + # schema keys, including `storageProfile.osDisk.diskSizeGB`. + vm_contract="$(az vm show --resource-group "${HBV4_RUNNERS_RG}" --name "${HBV4_VM_NAME}" \ + --output json --only-show-errors)" + if ! bash "${GITHUB_WORKSPACE}/ci/azure/scripts/validate-created-hbv4-vm.sh" \ + "${HBV4_VM_NAME}" "${HBV4_PROVIDER}" "${HBV4_ROLE}" \ + "${HBV4_INSTANCE_KEY}" "${HBV4_JIT_LABEL}" "${HBV4_UAMI_ID}" \ + <<< "${vm_contract}"; then + exit 1 + fi + nic_id="$(jq -r '.networkProfile.networkInterfaces[0].id // empty' \ + <<< "${vm_contract}")" + os_disk_id="$(jq -r '.storageProfile.osDisk.managedDisk.id // empty' \ + <<< "${vm_contract}")" + [[ -n "${nic_id}" && -n "${os_disk_id}" ]] || { + echo '::error::Azure VM omitted its exact NIC or OS disk identity.' + exit 1 + } + nic_contract="$(az network nic show --ids "${nic_id}" \ + --query '{subnet:ipConfigurations[0].subnet.id,publicIp:ipConfigurations[0].publicIPAddress.id,nsg:networkSecurityGroup.id,accelerated:enableAcceleratedNetworking}' \ + --output json --only-show-errors)" + jq -e --arg subnet_id "${HBV4_SUBNET_ID}" ' + (.subnet | ascii_downcase) == ($subnet_id | ascii_downcase) and + .publicIp == null and .nsg == null and .accelerated == true + ' <<< "${nic_contract}" >/dev/null || { + echo '::error::Azure NIC is not private, accelerated, and attached to the fixed subnet.' + exit 1 + } + for child_id in "${nic_id}" "${os_disk_id}"; do + az tag create --resource-id "${child_id}" --tags \ + project=GEOS \ + purpose=hbv4-tpl-ci \ + provider="${HBV4_PROVIDER}" \ + runnerRole="${HBV4_ROLE}" \ + githubRunId="${GITHUB_RUN_ID}" \ + runAttempt="${GITHUB_RUN_ATTEMPT}" \ + runnerInstanceKey="${HBV4_INSTANCE_KEY}" \ + runnerLabel="${HBV4_JIT_LABEL}" \ + logBlobPrefix="${HBV4_LOG_BLOB_PREFIX}" \ + createdAtUtc="${created_at}" \ + ttlHours=6 \ + --only-show-errors >/dev/null + done + echo "vm_name=${HBV4_VM_NAME}" >> "${GITHUB_OUTPUT}" + + - name: Require exact JIT runner online + id: health + shell: bash + env: + GH_TOKEN: ${{ steps.runner_app_token.outputs.token }} + HBV4_JIT_ID: ${{ steps.jit.outputs.runner_id }} + HBV4_JIT_NAME: ${{ steps.identity.outputs.runner_name }} + HBV4_JIT_LABEL: ${{ steps.identity.outputs.runner_label }} + HBV4_RUNNERS_RG: ${{ inputs.runner_resource_group }} + HBV4_VM_NAME: ${{ steps.create_vm.outputs.vm_name }} + run: | + set -euo pipefail + + [[ "${HBV4_JIT_ID}" =~ ^[1-9][0-9]*$ ]] || { + echo '::error::JIT runner ID is missing or invalid.' + exit 1 + } + + # The reviewed guest probe exposes only credential-free state and + # sanitized diagnostics. Run it once so terminal first-boot failures are + # surfaced promptly while GitHub remains the authoritative online check. + probe_guest_bootstrap() { + local guest_probe_script probe_output probe_status + guest_probe_script="$(<"${GITHUB_WORKSPACE}/ci/azure/scripts/probe-runner-bootstrap.sh")" + set +e + probe_output="$(timeout --signal=TERM --kill-after=10s 420s \ + az vm run-command invoke \ + --resource-group "${HBV4_RUNNERS_RG}" \ + --name "${HBV4_VM_NAME}" \ + --command-id RunShellScript \ + --scripts "${guest_probe_script}" \ + --query 'value[0].message' \ + --output tsv \ + --only-show-errors 2>/dev/null)" + probe_status=$? + set -e + if (( probe_status != 0 )); then + echo "Guest bootstrap monitor unavailable or timed out (${probe_status}); continuing bounded GitHub polling." + return 2 + fi + if grep -q 'GEOS_BOOTSTRAP_STATE=terminal' <<< "${probe_output}"; then + echo '::group::Sanitized terminal guest-bootstrap diagnostics' + printf '%s\n' "${probe_output}" + echo '::endgroup::' + return 1 + fi + if grep -Eq 'GEOS_BOOTSTRAP_STATE=(handoff|inconclusive)' <<< "${probe_output}"; then + return 0 + fi + echo 'Guest bootstrap monitor returned no recognized state; continuing bounded GitHub polling.' + return 2 + } + + deadline=$(( $(date +%s) + 900 )) + attempt=0 + guest_probe_pending=true + while (( $(date +%s) < deadline )); do + attempt=$(( attempt + 1 )) + pages="$(gh api --paginate --slurp \ + "repos/${GITHUB_REPOSITORY}/actions/runners?per_page=100" 2>/dev/null || true)" + if [[ -n "${pages}" ]] && jq -e \ + 'type == "array" and all(.[]; (.runners | type) == "array")' \ + <<< "${pages}" >/dev/null 2>&1; then + matches="$(jq -c --argjson id "${HBV4_JIT_ID}" \ + '[.[].runners[]? | select(.id == $id)]' <<< "${pages}")" + count="$(jq -r 'length' <<< "${matches}")" + if (( count > 1 )); then + echo "::error::Multiple GitHub runner registrations have JIT ID ${HBV4_JIT_ID}." + exit 1 + fi + if (( count == 1 )); then + observed_id="$(jq -r '.[0].id' <<< "${matches}")" + named="$(jq -r --arg name "${HBV4_JIT_NAME}" '.[0].name == $name' <<< "${matches}")" + online="$(jq -r '.[0].status == "online"' <<< "${matches}")" + labelled="$(jq -r --arg label "${HBV4_JIT_LABEL}" \ + 'any(.[0].labels[]?; .name == $label)' <<< "${matches}")" + echo "attempt ${attempt}: id=${observed_id} name_match=${named} status_online=${online} label_match=${labelled}" + if [[ "${named}" == true && "${online}" == true && + "${labelled}" == true && "${observed_id}" == "${HBV4_JIT_ID}" ]]; then + echo "Runner ${HBV4_JIT_NAME} (${HBV4_JIT_ID}) is online with ${HBV4_JIT_LABEL}." + exit 0 + fi + else + echo "attempt ${attempt}: runner ID ${HBV4_JIT_ID} not yet registered" + fi + else + echo "attempt ${attempt}: GitHub runner inventory unavailable" + fi + + if [[ "${guest_probe_pending}" == true ]]; then + guest_probe_pending=false + if probe_guest_bootstrap; then + : + else + probe_status=$? + if (( probe_status == 1 )); then + echo "::error::VM ${HBV4_VM_NAME} reached a terminal guest-bootstrap failure." + exit 1 + fi + fi + fi + sleep 15 + done + + echo "::error::Runner ${HBV4_JIT_NAME} (${HBV4_JIT_ID}) did not become online with label ${HBV4_JIT_LABEL} within 15 minutes." + exit 1 diff --git a/ci/azure/scripts/janitor-hbv4.sh b/ci/azure/scripts/janitor-hbv4.sh new file mode 100755 index 00000000..6d1f9792 --- /dev/null +++ b/ci/azure/scripts/janitor-hbv4.sh @@ -0,0 +1,484 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${AZURE_RUNNER_RESOURCE_GROUP:?}" +: "${AZURE_ACR_NAME:?}" +: "${GH_TOKEN:?}" +: "${ACTIONS_READ_TOKEN:?}" +: "${EVIDENCE_FILE:?}" + +install -d -m 0700 "$(dirname "${EVIDENCE_FILE}")" +: > "${EVIDENCE_FILE}" +now_epoch="$(date -u +%s)" + +record() { + local kind="$1" name="$2" action="$3" + jq -nc --arg kind "${kind}" --arg name "${name}" --arg action "${action}" \ + '{kind:$kind,name:$name,action:$action}' >> "${EVIDENCE_FILE}" +} + +is_expired() { + local created_at="$1" ttl_hours="$2" created_epoch + [[ "${created_at}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || return 2 + [[ "${ttl_hours}" =~ ^[1-9][0-9]*$ ]] || return 2 + (( ttl_hours <= 24 )) || return 2 + created_epoch="$(date -u -d "${created_at}" +%s)" || return 2 + (( now_epoch >= created_epoch + ttl_hours * 3600 )) +} + +vm_inventory="$(az vm list --resource-group "${AZURE_RUNNER_RESOURCE_GROUP}" \ + --output json --only-show-errors)" +jq -e 'type == "array"' <<< "${vm_inventory}" >/dev/null +mapfile -t vm_records < <(jq -c '.[]' <<< "${vm_inventory}") +for vm in "${vm_records[@]}"; do + name="$(jq -r '.name // empty' <<< "${vm}")" + project="$(jq -r '.tags.project // empty' <<< "${vm}")" + purpose="$(jq -r '.tags.purpose // empty' <<< "${vm}")" + provider="$(jq -r '.tags.provider // empty' <<< "${vm}")" + role="$(jq -r '.tags.runnerRole // empty' <<< "${vm}")" + run_id="$(jq -r '.tags.githubRunId // empty' <<< "${vm}")" + attempt="$(jq -r '.tags.runAttempt // empty' <<< "${vm}")" + created_at="$(jq -r '.tags.createdAtUtc // empty' <<< "${vm}")" + ttl_hours="$(jq -r '.tags.ttlHours // empty' <<< "${vm}")" + + [[ "${project}" == GEOS && "${purpose}" == hbv4-tpl-ci ]] || continue + case "${provider}" in openmpi|mpich) ;; *) record vm "${name}" preserved-invalid-tags; continue ;; esac + case "${role}" in builder|tester) ;; *) record vm "${name}" preserved-invalid-tags; continue ;; esac + [[ "${run_id}" =~ ^[1-9][0-9]*$ && "${attempt}" =~ ^[1-9][0-9]*$ ]] || { + record vm "${name}" preserved-invalid-tags + continue + } + expected="geos-hbv4-${run_id}-${attempt}-${provider}-${role}" + [[ "${name}" == "${expected}" ]] || { record vm "${name}" preserved-invalid-name; continue; } + if is_expired "${created_at}" "${ttl_hours}"; then + az vm delete --resource-group "${AZURE_RUNNER_RESOURCE_GROUP}" --name "${name}" \ + --yes --force-deletion true --only-show-errors + remaining="$(az vm list --resource-group "${AZURE_RUNNER_RESOURCE_GROUP}" \ + --query "[?name=='${name}'] | length(@)" --output tsv --only-show-errors)" + [[ "${remaining}" == 0 ]] || { echo "VM ${name} remains after janitor deletion" >&2; exit 1; } + record vm "${name}" deleted-expired + else + status=$? + if (( status == 2 )); then record vm "${name}" preserved-invalid-ttl; fi + fi +done + +# Reclaim explicitly tagged NIC/disk remnants left by a partial VM create or +# teardown. Other resource types and untagged resources are never candidates. +resource_inventory="$(az resource list --resource-group "${AZURE_RUNNER_RESOURCE_GROUP}" \ + --output json --only-show-errors)" +jq -e 'type == "array"' <<< "${resource_inventory}" >/dev/null +mapfile -t resource_records < <(jq -c '.[]' <<< "${resource_inventory}") +for resource in "${resource_records[@]}"; do + type="$(jq -r '.type // empty' <<< "${resource}")" + case "${type}" in + Microsoft.Network/networkInterfaces|Microsoft.Compute/disks) ;; + *) continue ;; + esac + id="$(jq -r '.id // empty' <<< "${resource}")" + name="$(jq -r '.name // empty' <<< "${resource}")" + project="$(jq -r '.tags.project // empty' <<< "${resource}")" + purpose="$(jq -r '.tags.purpose // empty' <<< "${resource}")" + provider="$(jq -r '.tags.provider // empty' <<< "${resource}")" + role="$(jq -r '.tags.runnerRole // empty' <<< "${resource}")" + run_id="$(jq -r '.tags.githubRunId // empty' <<< "${resource}")" + attempt="$(jq -r '.tags.runAttempt // empty' <<< "${resource}")" + created_at="$(jq -r '.tags.createdAtUtc // empty' <<< "${resource}")" + ttl_hours="$(jq -r '.tags.ttlHours // empty' <<< "${resource}")" + [[ "${project}" == GEOS && "${purpose}" == hbv4-tpl-ci ]] || continue + case "${provider}:${role}" in + openmpi:builder|openmpi:tester|mpich:builder|mpich:tester) ;; + *) record resource "${name}" preserved-invalid-tags; continue ;; + esac + [[ "${run_id}" =~ ^[1-9][0-9]*$ && "${attempt}" =~ ^[1-9][0-9]*$ ]] || { + record resource "${name}" preserved-invalid-tags + continue + } + runner_prefix="geos-hbv4-${run_id}-${attempt}-${provider}-${role}" + [[ "${name}" == "${runner_prefix}"* ]] || { + record resource "${name}" preserved-invalid-name + continue + } + if is_expired "${created_at}" "${ttl_hours}"; then + az resource delete --ids "${id}" --only-show-errors + record resource "${name}" deleted-expired + else + status=$? + if (( status == 2 )); then record resource "${name}" preserved-invalid-ttl; fi + fi +done + +# Remove an offline registration with no lifecycle-matched VM only when the +# workflow-run metadata encoded in its exact label proves that no VM can still +# be provisioned for it. A newly minted JIT runner is offline before az vm +# create completes, so VM absence alone is not evidence that it is orphaned. +# +# Labels are authoritative here. GitHub reserves RUNNER_NAME on hosted control +# runners; an older provision action accidentally passed that protected value +# to generate-jitconfig, producing names such as "GitHub Actions 1000003390". +# The same registrations retained their unique geos-hbv4 lifecycle labels. +vm_inventory="$(az vm list --resource-group "${AZURE_RUNNER_RESOURCE_GROUP}" \ + --output json --only-show-errors)" +jq -e 'type == "array" and all(.[]; (.name | type) == "string")' \ + <<< "${vm_inventory}" >/dev/null + +max_gh_attempts=4 +is_gh_not_found() { + grep -Eiq 'HTTP 404|Not Found' "$1" +} + +is_gh_retryable() { + grep -Eiq \ + 'HTTP (408|429|5[0-9][0-9])|secondary rate limit|rate limit|timeout|temporar' \ + "$1" +} + +validate_runner_inventory() { + jq -e ' + type == "array" and + all(.[]; + type == "object" and (.runners | type) == "array" and + all(.runners[]?; + (.id | type) == "number" and .id >= 1 and + .id == (.id | floor) and + (.name | type) == "string" and (.name | length) > 0 and + (.status | type) == "string" and + (.busy | type) == "boolean" and + (.labels | type) == "array" and + all(.labels[]?; (.name | type) == "string") + ) + ) + ' >/dev/null +} + +probe_runner_collection_access() { + local context="$1" response err_file attempt + for (( attempt = 1; attempt <= max_gh_attempts; attempt++ )); do + err_file="$(mktemp)" + if response="$(gh api \ + -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/actions/runners?per_page=1" \ + 2>"${err_file}")"; then + rm -f "${err_file}" + jq -e ' + type == "object" and + (.total_count | type) == "number" and + .total_count >= 0 and .total_count == (.total_count | floor) and + (.runners | type) == "array" + ' <<< "${response}" >/dev/null || { + echo "Cannot classify ${context}: runner collection response is malformed." >&2 + return 1 + } + return 0 + fi + if (( attempt < max_gh_attempts )) && is_gh_retryable "${err_file}"; then + echo "Runner collection probe attempt ${attempt} for ${context} was transient; retrying." >&2 + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + sleep $(( attempt * 10 )) + continue + fi + echo "Cannot classify ${context}: runner collection access failed." >&2 + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + return 1 + done + return 1 +} + +read_runner_inventory() { + local context="$1" response err_file attempt + for (( attempt = 1; attempt <= max_gh_attempts; attempt++ )); do + err_file="$(mktemp)" + if response="$(gh api --paginate --slurp \ + -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/actions/runners?per_page=100" \ + 2>"${err_file}")"; then + rm -f "${err_file}" + validate_runner_inventory <<< "${response}" || { + echo "${context} runner inventory is malformed." >&2 + return 1 + } + RUNNER_PAGES="${response}" + return 0 + fi + if (( attempt < max_gh_attempts )) && is_gh_retryable "${err_file}"; then + echo "Runner inventory attempt ${attempt} for ${context} was transient; retrying." >&2 + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + sleep $(( attempt * 10 )) + continue + fi + echo "Unable to read ${context} runner inventory." >&2 + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + return 1 + done + return 1 +} + +delete_and_verify_runner() { + local runner_id="$1" lifecycle_label="$2" runner_name="$3" + local delete_done=false err_file response attempt remaining_id remaining_label + + for (( attempt = 1; attempt <= max_gh_attempts; attempt++ )); do + err_file="$(mktemp)" + if gh api --method DELETE \ + -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/actions/runners/${runner_id}" \ + >/dev/null 2>"${err_file}"; then + rm -f "${err_file}" + delete_done=true + break + fi + if is_gh_not_found "${err_file}"; then + rm -f "${err_file}" + probe_runner_collection_access "runner DELETE 404 for ${runner_id}" || return 1 + delete_done=true + break + fi + if (( attempt < max_gh_attempts )) && is_gh_retryable "${err_file}"; then + echo "Runner DELETE attempt ${attempt} was transient; retrying." >&2 + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + sleep $(( attempt * 10 )) + continue + fi + echo "GitHub runner DELETE failed for ${runner_id}." >&2 + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + return 1 + done + [[ "${delete_done}" == true ]] || return 1 + + for (( attempt = 1; attempt <= max_gh_attempts; attempt++ )); do + err_file="$(mktemp)" + if response="$(gh api \ + -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/actions/runners/${runner_id}" \ + 2>"${err_file}")"; then + rm -f "${err_file}" + jq -e --argjson id "${runner_id}" 'type == "object" and .id == $id' \ + <<< "${response}" >/dev/null || { + echo "Runner GET returned a malformed identity for ${runner_id}." >&2 + return 1 + } + if (( attempt < max_gh_attempts )); then + sleep $(( attempt * 10 )) + continue + fi + echo "Runner ${runner_id} remains after janitor deletion." >&2 + return 1 + fi + if is_gh_not_found "${err_file}"; then + rm -f "${err_file}" + probe_runner_collection_access "runner verification 404 for ${runner_id}" || return 1 + break + fi + if (( attempt < max_gh_attempts )) && is_gh_retryable "${err_file}"; then + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + sleep $(( attempt * 10 )) + continue + fi + echo "Unable to verify runner ${runner_id} deletion." >&2 + sed -n '1,40p' "${err_file}" >&2 + rm -f "${err_file}" + return 1 + done + + read_runner_inventory deletion-verification || return 1 + remaining_id="$(jq -r --argjson id "${runner_id}" \ + '[.[].runners[]? | select(.id == $id)] | length' <<< "${RUNNER_PAGES}")" + remaining_label="$(jq -r --arg label "${lifecycle_label}" ' + [.[].runners[]? | + select([.labels[]?.name | select(. == $label)] | length > 0)] | length + ' <<< "${RUNNER_PAGES}")" + if (( remaining_id != 0 || remaining_label != 0 )); then + echo "Runner identity remains after janitor deletion: ${runner_name} / ${lifecycle_label}." >&2 + return 1 + fi +} + +RUNNER_PAGES='' +read_runner_inventory initial +runner_pages="${RUNNER_PAGES}" +mapfile -t runners < <(jq -c '.[].runners[]?' <<< "${runner_pages}") +lifecycle_pattern='^geos-hbv4-([1-9][0-9]*)-([1-9][0-9]*)-(openmpi|mpich)-(builder|tester)$' +for runner in "${runners[@]}"; do + runner_id="$(jq -r '.id // empty' <<< "${runner}")" + name="$(jq -r '.name // empty' <<< "${runner}")" + status="$(jq -r '.status // empty' <<< "${runner}")" + lifecycle_labels="$(jq -c --arg regex "${lifecycle_pattern}" \ + '[.labels[]?.name | select(test($regex))]' <<< "${runner}")" + lifecycle_label_count="$(jq -r 'length' <<< "${lifecycle_labels}")" + + if (( lifecycle_label_count == 0 )); then + if [[ "${name}" =~ ${lifecycle_pattern} ]]; then + record runner "${name}" preserved-missing-lifecycle-label + fi + continue + fi + if (( lifecycle_label_count != 1 )); then + record runner "${name}" preserved-ambiguous-lifecycle-label + continue + fi + lifecycle_label="$(jq -r '.[0]' <<< "${lifecycle_labels}")" + if [[ "${name}" =~ ${lifecycle_pattern} && "${name}" != "${lifecycle_label}" ]]; then + record runner "${name}" preserved-inconsistent-lifecycle-identity + continue + fi + label_owner_count="$(jq -r --arg label "${lifecycle_label}" ' + [.[].runners[]? | + select([.labels[]?.name | select(. == $label)] | length > 0)] | length + ' <<< "${runner_pages}")" + if (( label_owner_count != 1 )); then + record runner "${lifecycle_label}" preserved-ambiguous-lifecycle-label + continue + fi + [[ "${status}" == offline ]] || continue + [[ "${lifecycle_label}" =~ ${lifecycle_pattern} ]] + encoded_run_id="${BASH_REMATCH[1]}" + encoded_attempt="${BASH_REMATCH[2]}" + encoded_provider="${BASH_REMATCH[3]}" + encoded_role="${BASH_REMATCH[4]}" + has_vm="$(jq -r \ + --arg name "${lifecycle_label}" \ + --arg run_id "${encoded_run_id}" \ + --arg attempt "${encoded_attempt}" \ + --arg provider "${encoded_provider}" \ + --arg role "${encoded_role}" ' + [ + .[] + | select( + .name == $name or + ( + (.tags | type) == "object" and + .tags.project == "GEOS" and + .tags.purpose == "hbv4-tpl-ci" and + (.tags.githubRunId | tostring) == $run_id and + (.tags.runAttempt | tostring) == $attempt and + .tags.provider == $provider and + .tags.runnerRole == $role + ) + ) + ] | length + ' <<< "${vm_inventory}")" + (( has_vm == 0 )) || continue + + set +e + workflow_run="$(GH_TOKEN="${ACTIONS_READ_TOKEN}" gh api \ + -H 'Accept: application/vnd.github+json' \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${encoded_run_id}" 2>/dev/null)" + workflow_run_status=$? + set -e + if (( workflow_run_status != 0 )); then + record runner "${lifecycle_label}" preserved-unverifiable-workflow-run + continue + fi + + if ! jq -e \ + --arg repository "${GITHUB_REPOSITORY}" \ + --argjson run_id "${encoded_run_id}" ' + type == "object" and + (.id | type) == "number" and .id == $run_id and + (.repository.full_name | type) == "string" and + .repository.full_name == $repository and + (.path | type) == "string" and + .path == ".github/workflows/docker_build_tpls_hbv4.yml" and + (.event == "workflow_dispatch" or .event == "pull_request_target") and + (.run_attempt | type) == "number" and + .run_attempt >= 1 and .run_attempt == (.run_attempt | floor) and + (.status | type) == "string" and + (.conclusion == null or (.conclusion | type) == "string") + ' <<< "${workflow_run}" >/dev/null; then + record runner "${lifecycle_label}" preserved-unverifiable-workflow-run + continue + fi + + live_attempt="$(jq -r '.run_attempt' <<< "${workflow_run}")" + run_status="$(jq -r '.status' <<< "${workflow_run}")" + conclusion="$(jq -r '.conclusion // empty' <<< "${workflow_run}")" + delete_action='' + if (( encoded_attempt < live_attempt )); then + delete_action=deleted-superseded-attempt-orphan + elif (( encoded_attempt > live_attempt )); then + record runner "${lifecycle_label}" preserved-inconsistent-workflow-run + continue + elif [[ "${run_status}" == completed && -n "${conclusion}" ]]; then + delete_action=deleted-completed-orphan + elif [[ "${run_status}" =~ ^(queued|in_progress|requested|waiting|pending)$ && -z "${conclusion}" ]]; then + record runner "${lifecycle_label}" preserved-active-workflow-run + continue + else + record runner "${lifecycle_label}" preserved-inconsistent-workflow-run + continue + fi + + delete_and_verify_runner "${runner_id}" "${lifecycle_label}" "${name}" + record runner "${lifecycle_label}" "${delete_action}" +done + +# Failed or ambiguous candidates are intentionally recoverable for seven days. +repository_inventory="$(az acr repository list --name "${AZURE_ACR_NAME}" \ + --output json --only-show-errors)" +jq -e 'type == "array" and all(.[]; type == "string")' \ + <<< "${repository_inventory}" >/dev/null +mapfile -t candidate_repositories < <(jq -r '.[] | select(test("^geos/hbv4-tpl-candidates/(openmpi|mpich)/run-[1-9][0-9]*-attempt-[1-9][0-9]*$"))' \ + <<< "${repository_inventory}") +for repository in "${candidate_repositories[@]}"; do + repository_metadata="$(az acr repository show --name "${AZURE_ACR_NAME}" \ + --repository "${repository}" --output json --only-show-errors)" + if ! jq -e --arg repository "${repository}" ' + type == "object" and + (has("imageName") or has("name")) and + ((has("imageName") | not) or + ((.imageName | type) == "string" and .imageName == $repository)) and + ((has("name") | not) or + ((.name | type) == "string" and .name == $repository)) and + ((.lastUpdateTime | type) == "string" and + (.lastUpdateTime | length) > 0) + ' <<< "${repository_metadata}" >/dev/null; then + record candidate "${repository}" preserved-malformed-repository-metadata + echo "Candidate repository metadata is malformed for ${repository}; preserving it." >&2 + exit 1 + fi + updated="$(jq -r '.lastUpdateTime' <<< "${repository_metadata}")" + if ! updated_epoch="$(date -u -d "${updated}" +%s)"; then + record candidate "${repository}" preserved-invalid-repository-time + echo "Candidate repository timestamp is malformed for ${repository}; preserving it." >&2 + exit 1 + fi + if (( now_epoch >= updated_epoch + 7 * 24 * 3600 )); then + az acr repository delete --name "${AZURE_ACR_NAME}" \ + --repository "${repository}" --yes --only-show-errors + repositories_after_delete="$(az acr repository list --name "${AZURE_ACR_NAME}" \ + --output json --only-show-errors)" + jq -e --arg repository "${repository}" ' + type == "array" and all(.[]; type == "string") and + all(.[]; . != $repository) + ' <<< "${repositories_after_delete}" >/dev/null || { + echo "Candidate repository remains after janitor deletion: ${repository}" >&2 + exit 1 + } + record candidate "${repository}" deleted-seven-day + else + tags="$(az acr repository show-tags --name "${AZURE_ACR_NAME}" \ + --repository "${repository}" --detail --output json --only-show-errors)" + if jq -e 'type == "array" and length == 1 and .[0].name == "candidate"' \ + <<< "${tags}" >/dev/null; then + record candidate "${repository}" preserved-within-seven-day + else + record candidate "${repository}" preserved-ambiguous-within-seven-day + fi + fi +done + +# A successful collection read after mutation is the verification boundary. +final_vms="$(az vm list --resource-group "${AZURE_RUNNER_RESOURCE_GROUP}" --output json --only-show-errors)" +final_repositories="$(az acr repository list --name "${AZURE_ACR_NAME}" --output json --only-show-errors)" +final_runners="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runners?per_page=1")" +jq -e 'type == "array"' <<< "${final_vms}" >/dev/null +jq -e 'type == "array"' <<< "${final_repositories}" >/dev/null +jq -e 'type == "object" and (.runners | type) == "array"' <<< "${final_runners}" >/dev/null diff --git a/ci/azure/scripts/probe-runner-bootstrap.sh b/ci/azure/scripts/probe-runner-bootstrap.sh new file mode 100755 index 00000000..65c64c24 --- /dev/null +++ b/ci/azure/scripts/probe-runner-bootstrap.sh @@ -0,0 +1,342 @@ +#!/usr/bin/env bash + +# @file probe-runner-bootstrap.sh +# +# Credential-free guest probe for an allocated Azure runner VM. +# +# The provisioning control job sends this reviewed script through Azure Run +# Command while GitHub still reports the JIT runner offline. It does not decide +# success: GitHub's online runner record plus the unique label remains +# authoritative. Its only purpose is to distinguish a bootstrap that is still +# progressing from one that has already terminated, so the control job can stop +# billing promptly and enter unconditional cleanup. +# +# Implementation workflow (single entry point: main): +# main() +# - read each atomically replaced status generation through one file +# descriptor and validate its fixed schema as data, never by sourcing it; +# - give a validated terminal status precedence over cloud-final failure, +# then give cloud-final failure precedence over a running +# handoff/inconclusive marker. A failed cloud-final causes exactly one +# status re-read so an EXIT-trap record published during the service check +# retains its precise phase and exit code; +# - return as soon as bootstrap terminates or hands control to a live runner +# process, while a bounded inconclusive result preserves the caller's +# original slow-boot allowance; +# - call emit_terminal_diagnostics() for failed, exited, or malformed state, +# then emit the machine-readable terminal marker last so Azure's 4-KiB +# returned-output limit cannot truncate it. +# emit_terminal_diagnostics() +# - collect only validated status fields, non-secret structured +# storage/service state, and the reviewed bootstrap phase log; +# - cap the diagnostic prefix to 3.2 KiB. Never inspect customData, +# arbitrary cloud-init output, environment variables, process arguments, +# runner `_diag`, or shell history because those sources can contain the +# one-time JIT credential. +# +# The explicit test-mode root redirects file reads into a fixture directory. +# Production ignores every path override unless GEOS_BOOTSTRAP_PROBE_TEST_MODE +# is exactly 1. + +set -u + +if [[ "${GEOS_BOOTSTRAP_PROBE_TEST_MODE:-0}" == "1" ]]; then + PROBE_ROOT="${GEOS_BOOTSTRAP_PROBE_TEST_ROOT:?test root is required}" + STATUS_FILE="${PROBE_ROOT}/run/geos-runner-bootstrap.status" + BOOT_LOG="${PROBE_ROOT}/var/log/geos-runner-bootstrap.log" + MDSTAT_PATH="${PROBE_ROOT}/proc/mdstat" + MOUNT_TARGET="${PROBE_ROOT}/mnt/hbv4-local" + MONITOR_SECONDS="${GEOS_BOOTSTRAP_PROBE_TEST_TIMEOUT_SECONDS:-1}" + POLL_SECONDS="${GEOS_BOOTSTRAP_PROBE_TEST_POLL_SECONDS:-1}" +else + STATUS_FILE="/run/geos-runner-bootstrap.status" + BOOT_LOG="/var/log/geos-runner-bootstrap.log" + MDSTAT_PATH="/proc/mdstat" + MOUNT_TARGET="/mnt/hbv4-local" + MONITOR_SECONDS=300 + POLL_SECONDS=5 +fi +readonly STATUS_FILE BOOT_LOG MDSTAT_PATH MOUNT_TARGET +readonly MONITOR_SECONDS POLL_SECONDS + +[[ "${MONITOR_SECONDS}" =~ ^[1-9][0-9]*$ ]] || + { + echo "GEOS_BOOTSTRAP_STATE=terminal source=probe phase=invalid-monitor-timeout exit_code=1" + exit 0 + } +[[ "${POLL_SECONDS}" =~ ^[1-9][0-9]*$ ]] || + { + echo "GEOS_BOOTSTRAP_STATE=terminal source=probe phase=invalid-poll-interval exit_code=1" + exit 0 + } + +# This string is populated only from fields that passed the fixed-schema +# allowlists below. In particular, malformed status-file contents are never +# copied into workflow logs. +STATUS_DIAGNOSTIC="bootstrap status was unavailable or invalid" +STATUS_RECORD_KIND="missing" +STATUS_INVALID_REASON="invalid-record" +STATUS_VERSION="" +STATUS_STATE="" +STATUS_PHASE="" +STATUS_EXIT_CODE="" +STATUS_UPDATED_AT="" + +# emit_terminal_diagnostics: return a bounded, credential-free failure report. +# +# Entry state: a terminal condition has already been proved. Exit state: at +# most the last 3.2 KiB of the approved sources has been written. The caller +# must emit its terminal marker after this function returns. +emit_terminal_diagnostics() +{ + local diagnostic_file + diagnostic_file="$(mktemp)" + { + echo "--- validated bootstrap status (credential-free) ---" + printf "%s\n" "${STATUS_DIAGNOSTIC}" + + echo "--- non-secret guest storage state ---" + lsblk -e 7 -o NAME,KNAME,PATH,TYPE,SIZE,FSTYPE,LABEL,MOUNTPOINTS,MODEL \ + 2>/dev/null || true + cat "${MDSTAT_PATH}" 2>/dev/null || true + findmnt --target "${MOUNT_TARGET}" 2>/dev/null || true + systemctl show cloud-final.service docker.service \ + --property=Id \ + --property=ActiveState \ + --property=SubState \ + --property=Result \ + --property=ExecMainCode \ + --property=ExecMainStatus \ + 2>/dev/null || true + + echo "--- GEOS bootstrap phase log (credential-free) ---" + if [[ -r "${BOOT_LOG}" ]]; then + tail -n 80 "${BOOT_LOG}" + else + echo "${BOOT_LOG} is unavailable" + fi + } > "${diagnostic_file}" + tail -c 3200 "${diagnostic_file}" + rm -f "${diagnostic_file}" +} + +# read_status_record: capture and validate one complete status generation. +# +# Atomic replacement guarantees that one open descriptor continues reading the +# same inode even if the writer publishes a newer phase concurrently. The +# single-stream parser therefore cannot combine fields from different +# generations. It accepts the writer's five keys in any order, rejects +# duplicate/missing/extra fields without logging their contents, and exposes +# values only after every fixed allowlist has passed. On return, +# STATUS_RECORD_KIND is exactly missing, invalid, running, failed, or exited; +# STATUS_INVALID_REASON retains the sanitized schema/value distinction when the +# classification is invalid. +read_status_record() +{ + local line + local version_count=0 + local state_count=0 + local phase_count=0 + local exit_code_count=0 + local updated_at_count=0 + local -a status_lines=() + + STATUS_DIAGNOSTIC="bootstrap status was unavailable or invalid" + STATUS_RECORD_KIND="missing" + STATUS_INVALID_REASON="invalid-record" + STATUS_VERSION="" + STATUS_STATE="" + STATUS_PHASE="" + STATUS_EXIT_CODE="" + STATUS_UPDATED_AT="" + + [[ -r "${STATUS_FILE}" ]] || return 0 + if ! while IFS= read -r line || [[ -n "${line}" ]]; do + status_lines+=("${line}") + done < "${STATUS_FILE}"; then + return 0 + fi + + if (( ${#status_lines[@]} != 5 )); then + STATUS_RECORD_KIND="invalid" + return 0 + fi + + for line in "${status_lines[@]}"; do + case "${line}" in + version=*) + version_count=$(( version_count + 1 )) + STATUS_VERSION="${line#version=}" + ;; + state=*) + state_count=$(( state_count + 1 )) + STATUS_STATE="${line#state=}" + ;; + phase=*) + phase_count=$(( phase_count + 1 )) + STATUS_PHASE="${line#phase=}" + ;; + exit_code=*) + exit_code_count=$(( exit_code_count + 1 )) + STATUS_EXIT_CODE="${line#exit_code=}" + ;; + updated_at=*) + updated_at_count=$(( updated_at_count + 1 )) + STATUS_UPDATED_AT="${line#updated_at=}" + ;; + *) + STATUS_RECORD_KIND="invalid" + return 0 + ;; + esac + done + + if (( version_count != 1 || + state_count != 1 || + phase_count != 1 || + exit_code_count != 1 || + updated_at_count != 1 )); then + STATUS_RECORD_KIND="invalid" + return 0 + fi + + if [[ "${STATUS_VERSION}" != "1" ]] || + ! printf "%s\n" "${STATUS_STATE}" | grep -Eq '^(running|failed|exited)$' || + ! printf "%s\n" "${STATUS_PHASE}" | grep -Eq '^[a-z0-9-]+$' || + ! printf "%s\n" "${STATUS_EXIT_CODE}" | grep -Eq '^[0-9]+$' || + ! printf "%s\n" "${STATUS_UPDATED_AT}" | + grep -Eq '^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$'; then + STATUS_RECORD_KIND="invalid" + STATUS_INVALID_REASON="invalid-value" + return 0 + fi + + STATUS_RECORD_KIND="${STATUS_STATE}" + STATUS_DIAGNOSTIC="version=${STATUS_VERSION} state=${STATUS_STATE} phase=${STATUS_PHASE} exit_code=${STATUS_EXIT_CODE} updated_at=${STATUS_UPDATED_AT}" +} + +# emit_status_terminal_if_present: apply status-only terminal precedence. +# +# Valid terminal states preserve the bootstrap's precise phase and exit code. +# Malformed records and contradictory state/exit-code pairs remain sanitized +# status-file failures. A return of one means the current record is either +# absent or validly running and therefore requires the cloud-final decision. +emit_status_terminal_if_present() +{ + case "${STATUS_RECORD_KIND}" in + invalid) + emit_terminal_diagnostics + printf "GEOS_BOOTSTRAP_STATE=terminal source=status-file phase=%s exit_code=1\n" \ + "${STATUS_INVALID_REASON}" + return 0 + ;; + missing) + return 1 + ;; + esac + + case "${STATUS_RECORD_KIND}" in + running) + if [[ "${STATUS_EXIT_CODE}" != "0" ]]; then + emit_terminal_diagnostics + echo "GEOS_BOOTSTRAP_STATE=terminal source=status-file phase=invalid-running-exit-code exit_code=1" + return 0 + fi + return 1 + ;; + failed) + if [[ "${STATUS_EXIT_CODE}" == "0" ]]; then + emit_terminal_diagnostics + echo "GEOS_BOOTSTRAP_STATE=terminal source=status-file phase=invalid-failed-exit-code exit_code=1" + return 0 + fi + emit_terminal_diagnostics + printf "\nGEOS_BOOTSTRAP_STATE=terminal source=status-file phase=%s exit_code=%s\n" \ + "${STATUS_PHASE}" "${STATUS_EXIT_CODE}" + return 0 + ;; + exited) + if [[ "${STATUS_EXIT_CODE}" != "0" ]]; then + emit_terminal_diagnostics + echo "GEOS_BOOTSTRAP_STATE=terminal source=status-file phase=invalid-exited-exit-code exit_code=1" + return 0 + fi + emit_terminal_diagnostics + printf "\nGEOS_BOOTSTRAP_STATE=terminal source=status-file phase=%s exit_code=0\n" \ + "${STATUS_PHASE}" + return 0 + ;; + esac + + emit_terminal_diagnostics + echo "GEOS_BOOTSTRAP_STATE=terminal source=status-file phase=invalid-state exit_code=1" + return 0 +} + +# cloud_final_failed: consume only cloud-init's fixed terminal classification. +# +# A running bootstrap keeps cloud-final active, so a failed service means the +# custom-data supervisor has terminated. The caller re-reads status after this +# result to preserve any precise EXIT-trap record published during these checks. +cloud_final_failed() +{ + local cloud_init_summary + cloud_init_summary="$(cloud-init status 2>/dev/null || true)" + systemctl is-failed --quiet cloud-final.service 2>/dev/null || + printf "%s\n" "${cloud_init_summary}" | + grep -Eq '^status:[[:space:]]*error([[:space:]]|$)' +} + +main() +{ + local deadline + local last_running_marker="GEOS_BOOTSTRAP_STATE=inconclusive last_state=missing" + + deadline=$(( SECONDS + MONITOR_SECONDS )) + while true; do + read_status_record + if emit_status_terminal_if_present; then + return 0 + fi + + if [[ "${STATUS_RECORD_KIND}" == "running" ]]; then + last_running_marker="GEOS_BOOTSTRAP_STATE=inconclusive last_state=running phase=${STATUS_PHASE} updated_at=${STATUS_UPDATED_AT}" + fi + + if cloud_final_failed; then + # The EXIT trap can publish a precise terminal record while cloud-final is + # transitioning to failed. Re-read exactly once and prefer that record; + # an unchanged running record is stale because its supervisor has ended. + read_status_record + if emit_status_terminal_if_present; then + return 0 + fi + if [[ "${STATUS_RECORD_KIND}" == "running" ]]; then + STATUS_DIAGNOSTIC="${STATUS_DIAGNOSTIC}; cloud-final reported a terminal failure while this running record remained" + else + STATUS_DIAGNOSTIC="bootstrap status unavailable; cloud-final reported a terminal failure" + fi + emit_terminal_diagnostics + echo "GEOS_BOOTSTRAP_STATE=terminal source=cloud-init phase=cloud-final exit_code=1" + return 0 + fi + + # runner-bootstrap.sh publishes this phase only after the runner child has + # survived its launch guard. Handoff is valid only while cloud-final remains + # active; a failed supervisor outranks even this otherwise-live child state. + if [[ "${STATUS_RECORD_KIND}" == "running" && + "${STATUS_PHASE}" == "runner-agent" ]]; then + printf "GEOS_BOOTSTRAP_STATE=handoff phase=%s updated_at=%s\n" \ + "${STATUS_PHASE}" "${STATUS_UPDATED_AT}" + return 0 + fi + + if (( SECONDS >= deadline )); then + printf "%s\n" "${last_running_marker}" + return 0 + fi + sleep "${POLL_SECONDS}" + done +} + +main "$@" diff --git a/ci/azure/scripts/render-runner-bootstrap.py b/ci/azure/scripts/render-runner-bootstrap.py new file mode 100755 index 00000000..becf62e2 --- /dev/null +++ b/ci/azure/scripts/render-runner-bootstrap.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Render the reviewed HBv4 cloud-init payload without shell evaluation.""" + +from __future__ import annotations + +import base64 +import gzip +import hashlib +import os +import pathlib +import re +import sys +import textwrap + + +AZURE_CUSTOM_DATA_LIMIT = 65_535 + + +def render_payload(source: str) -> bytes: + """Substitute the allowlisted single-line values into the reviewed script.""" + + + replacements = { + "@@RUNNER_ENCODED_JITCONFIG@@": "RUNNER_ENCODED_JITCONFIG", + "@@RUNNER_VERSION@@": "RUNNER_VERSION", + "@@RUNNER_TARBALL_SHA256@@": "RUNNER_TARBALL_SHA256", + "@@ACR_NAME@@": "ACR_NAME", + "@@RUNNER_VM_UAMI_CLIENT_ID@@": "RUNNER_VM_UAMI_CLIENT_ID", + "@@RUNNER_STORAGE_PROFILE@@": "RUNNER_STORAGE_PROFILE", + "@@STORAGE_HELPER_SHA256@@": "STORAGE_HELPER_SHA256", + "@@STORAGE_HELPER_GZIP_BASE64@@": "STORAGE_HELPER_GZIP_BASE64", + "@@HOST_VALIDATOR_SHA256@@": "HOST_VALIDATOR_SHA256", + "@@HOST_VALIDATOR_GZIP_BASE64@@": "HOST_VALIDATOR_GZIP_BASE64", + "@@CONTAINER_VALIDATOR_SHA256@@": "CONTAINER_VALIDATOR_SHA256", + "@@CONTAINER_VALIDATOR_GZIP_BASE64@@": "CONTAINER_VALIDATOR_GZIP_BASE64", + "@@PARALLEL_HDF5_SOURCE_SHA256@@": "PARALLEL_HDF5_SOURCE_SHA256", + "@@PARALLEL_HDF5_SOURCE_GZIP_BASE64@@": "PARALLEL_HDF5_SOURCE_GZIP_BASE64", + } + for marker, variable in replacements.items(): + if source.count(marker) != 1: + raise SystemExit(f"{marker} must occur exactly once") + value = os.environ.get(variable, "") + if not value: + raise SystemExit(f"required render variable {variable} is empty") + if "\x00" in value or "\n" in value or "\r" in value: + raise SystemExit(f"render variable {variable} is not single-line") + source = source.replace(marker, value) + + payload = source.encode("ascii") + if re.search(rb"@@[A-Z0-9_]+@@", payload): + raise SystemExit("unrecognized template placeholder remains in bootstrap payload") + return payload + + +def wrap_payload(payload: bytes) -> bytes: + """Create a small, deterministic, authenticated first-boot decoder.""" + + payload_sha256 = hashlib.sha256(payload).hexdigest() + encoded = base64.b64encode(gzip.compress(payload, compresslevel=9, mtime=0)).decode("ascii") + wrapped_encoded = "\n".join(textwrap.wrap(encoded, 120)) + wrapper = f"""#!/usr/bin/env bash +set -Eeuo pipefail + +readonly PAYLOAD_SHA256='{payload_sha256}' +payload_path="$(mktemp /var/tmp/geos-hbv4-bootstrap.XXXXXX)" +cleanup() {{ rm -f "${{payload_path}}"; }} +trap cleanup EXIT + +if ! base64 --decode <<'__GEOS_HBV4_BOOTSTRAP__' | gzip -dc > "${{payload_path}}" +{wrapped_encoded} +__GEOS_HBV4_BOOTSTRAP__ +then + echo 'GEOS HBv4 bootstrap payload decode failed' >&2 + exit 1 +fi +actual_sha256="$(sha256sum "${{payload_path}}" | awk '{{print $1}}')" +if [[ "${{actual_sha256}}" != "${{PAYLOAD_SHA256}}" ]]; then + echo 'GEOS HBv4 bootstrap payload SHA-256 mismatch' >&2 + exit 1 +fi +chmod 0700 "${{payload_path}}" +"${{payload_path}}" +""".encode("ascii") + if len(wrapper) > AZURE_CUSTOM_DATA_LIMIT: + raise SystemExit( + f"Azure custom data is {len(wrapper)} bytes; limit is {AZURE_CUSTOM_DATA_LIMIT}" + ) + return wrapper + + +def main() -> int: + if len(sys.argv) != 3: + raise SystemExit("usage: render-runner-bootstrap.py INPUT OUTPUT") + + source = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") + payload = render_payload(source) + pathlib.Path(sys.argv[2]).write_bytes(wrap_payload(payload)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci/azure/scripts/runner-bootstrap.sh b/ci/azure/scripts/runner-bootstrap.sh new file mode 100755 index 00000000..25fe8e0d --- /dev/null +++ b/ci/azure/scripts/runner-bootstrap.sh @@ -0,0 +1,954 @@ +#!/usr/bin/env bash + +# @file runner-bootstrap.sh +# +# First-boot bootstrap for a GEOS Azure ephemeral CI runner VM. +# +# KEEP THIS FILE ASCII-ONLY. It is passed verbatim as `az vm create --custom-data`, +# which encodes the payload as latin-1 and FAILS on any non-ASCII byte (e.g. an +# em-dash or section sign): "'latin-1' codec can't encode character ...". +# +# This script is the VM `--custom-data` payload passed by the `provision` job of +# the in-workflow trio (.github/workflows/azure-job.yml, see +# plans/azure-ephemeral-runners.md sec 3, sec 5.5, sec 9). On the +# caller-selected Ubuntu Marketplace image (a GEOS-specific pre-baked image is +# deferred -- plan sec 6/sec 10 M1a), the Azure VM agent runs this once, as +# root, via cloud-init's "script" custom-data path on first boot. Its job is to +# turn the VM into a single-use, +# just-in-time (JIT) GitHub Actions runner that pulls the GEOS build container +# from the foundation ACR, runs exactly ONE CI job, and exits. +# +# Why this lives in a committed file rather than being generated inline: the +# bootstrap is security-sensitive shell that runs as root on a VM that holds the +# runner-VM managed identity (reachable via IMDS by the CI job's own code), so it +# must be reviewable, shellcheck-clean, and version-controlled. The `provision` +# job templates per-run values into a COPY of this file before handing it to +# `az vm create` (see the "Templated placeholders" contract below); the committed +# file itself contains only inert placeholder tokens and never a real credential. +# +# Templated placeholders (substituted by the provision job, never committed live): +# RUNNER_ENCODED_JITCONFIG -- the base64 `encoded_jit_config` returned by +# GitHub's generate-jitconfig endpoint. This is +# a ONE-TIME runner credential (see the +# sensitivity note below). Required. +# RUNNER_VERSION -- the pinned actions/runner version, e.g. +# 2.335.1. Pinned + SHA-verified (plan gotcha). +# RUNNER_TARBALL_SHA256 -- the expected SHA-256 of the linux-x64 runner +# tarball for RUNNER_VERSION. Verified +# before the tarball is unpacked/executed. +# ACR_NAME -- the ACR registry NAME (not login server), +# e.g. geoscisc2053650e, for `az acr login`. +# RUNNER_VM_UAMI_CLIENT_ID -- the client id of the user-assigned managed +# identity id-geos-runner-vm attached to this +# VM. REQUIRED for `az login --identity` because +# a bare --identity resolves to the system +# identity (which this VM does not have) and +# fails (plan sec 8.3 gotcha). +# RUNNER_STORAGE_PROFILE -- "none" for the historical OS-disk layout or +# "hbv4-nvme-raid0" for the reviewed HB176 +# two-device local-NVMe layout. +# STORAGE_HELPER_GZIP_BASE64 -- gzip-compressed, base64-encoded bytes of the +# committed setup-hbv4-local-nvme.sh helper. +# STORAGE_HELPER_SHA256 -- SHA-256 of the uncompressed helper. The VM +# verifies this before installing it as root. +# HOST_VALIDATOR_*, CONTAINER_VALIDATOR_*, PARALLEL_HDF5_SOURCE_* +# -- independently hashed gzip/base64 payloads +# from the trusted default-branch control SHA. +# +# Implementation workflow (single entry point: main, run once at first boot): +# main() +# log()/die() -- timestamped logging to a boot log file + +# stdout (captured by cloud-init / serial +# console for post-mortem, plan sec 9.5). +# set_bootstrap_phase() -- atomically publish a credential-free +# phase/state record for the control-plane +# post-boot health check. +# record_bootstrap_exit() -- EXIT trap that converts any early shell +# termination into a sanitized terminal +# state without recording the failed command, +# environment, arguments, or JIT credential. +# require_placeholders() -- fail fast if any @@...@@ token survived (a +# templating bug); refusing to proceed avoids +# a half-configured VM that bills while never +# coming online (the runner would then only be +# reaped by the janitor's hard max-age cap). +# install_base_packages() -- apt-get the host tooling the runner agent + +# the GEOS docker-run build model need (curl, +# jq, git, git-lfs, ca-certs). The GEOS +# TOOLCHAIN itself lives in the container +# image; only this thin host layer is here. +# install_storage_helper() -- decode the embedded RAID helper, verify its +# independent SHA-256, and install it as root. +# configure_runner_storage() -- for the HBv4 profile only, synchronously +# create/mount the reviewed RAID-0 array, +# place runner state on that mount, and create +# the dedicated scratch directory that the +# workload binds over container /tmp. +# install_docker() -- install Docker Engine from Docker's apt repo +# and enable the daemon, because the workload +# job runs the build as `docker run` on the +# host (plan sec 2/sec 3: not a job-level container, +# so ARC/DinD is rejected). +# ensure_azure_cli() -- ensure `az` + the azcopy binary exist (Blob +# artifact/baseline transfer, plan sec 9.3). The +# CLI may already be present on some images; +# installation is idempotent. +# azure_login_and_acr_login() -- `az login --identity --client-id ` +# then `az acr login --name ` so the +# subsequent in-job `docker pull` from ACR is +# authenticated by the VM's AcrPull role +# without any stored registry secret +# (plan sec 8.1: AcrPull RBAC alone does not +# authenticate docker; an `az acr login` is +# required, best done here at boot). +# install_runner_agent() -- download the PINNED runner tarball, VERIFY +# its SHA-256 before touching it, unpack it as +# a non-root runner user, and install the +# agent's OS dependencies. +# validate_runner_storage() -- immediately before the foreground runner, +# prove the HB mount/RAID/XFS/capacity, +# runner-home, and container-scratch +# invariants, including RAID-backed Docker. +# run_runner_ephemeral() -- start the agent with `--jitconfig` so it +# registers JIT, accepts exactly one job, and +# exits (ephemeral). No registration token, +# no long-lived runner. +# +# Sensitivity of the encoded jitconfig (accepted, documented): +# The base64 jitconfig embedded here is a one-time credential that lets this VM +# register as the named JIT runner and receive one job. Because it is delivered +# as custom-data, it is readable from inside the VM via IMDS +# (http://169.254.169.254/metadata/instance/compute/customData) by anything that +# can reach IMDS, including the CI job's own code. That exposure is ACCEPTED for +# a SINGLE-JOB EPHEMERAL VM: the credential is consumed on first registration, +# the runner is removed from GitHub once it accepts its one job (JIT +# auto-removal), the `cleanup` job deletes both the VM and the registration, and +# the janitor is the backstop. It is NOT a standing secret and grants only the +# ability to be this one runner. Do not reuse this VM for a second job, and do +# not widen the runner-VM identity beyond AcrPull + container-scoped Blob. + +set -Eeuo pipefail + +# Per-run values templated in by the provision job. Kept as shell variables (not +# scattered inline) so require_placeholders() can validate them in one place and +# so a future maintainer sees the full substitution contract at the top of the +# executable body. The committed values are inert @@...@@ tokens. +RUNNER_ENCODED_JITCONFIG="@@RUNNER_ENCODED_JITCONFIG@@" +RUNNER_VERSION="@@RUNNER_VERSION@@" +RUNNER_TARBALL_SHA256="@@RUNNER_TARBALL_SHA256@@" +ACR_NAME="@@ACR_NAME@@" +RUNNER_VM_UAMI_CLIENT_ID="@@RUNNER_VM_UAMI_CLIENT_ID@@" +RUNNER_STORAGE_PROFILE="@@RUNNER_STORAGE_PROFILE@@" +STORAGE_HELPER_GZIP_BASE64="@@STORAGE_HELPER_GZIP_BASE64@@" +STORAGE_HELPER_SHA256="@@STORAGE_HELPER_SHA256@@" +HOST_VALIDATOR_GZIP_BASE64="@@HOST_VALIDATOR_GZIP_BASE64@@" +HOST_VALIDATOR_SHA256="@@HOST_VALIDATOR_SHA256@@" +CONTAINER_VALIDATOR_GZIP_BASE64="@@CONTAINER_VALIDATOR_GZIP_BASE64@@" +CONTAINER_VALIDATOR_SHA256="@@CONTAINER_VALIDATOR_SHA256@@" +PARALLEL_HDF5_SOURCE_GZIP_BASE64="@@PARALLEL_HDF5_SOURCE_GZIP_BASE64@@" +PARALLEL_HDF5_SOURCE_SHA256="@@PARALLEL_HDF5_SOURCE_SHA256@@" + +# Non-root account the runner agent runs under. The agent refuses to run its +# service as root; jobs that need root use sudo/docker. The default preserves +# the historical path. The HBv4 profile changes this to the mounted local NVMe +# before any runner files are installed, while JIT work_folder remains the +# relative "_work" under this directory. +RUNNER_USER="ghrunner" +RUNNER_HOME="/opt/actions-runner" +HBV4_MOUNT_ROOT="/mnt/hbv4-local" +HBV4_MD_DEVICE="/dev/md/hbv4-local" +HBV4_CONTAINER_TMP="${HBV4_MOUNT_ROOT}/container-tmp" +STORAGE_HELPER_PATH="/usr/local/sbin/setup-hbv4-local-nvme" +TRUSTED_VALIDATION_ROOT="/opt/geos-ci/scripts/hbv4" + +# Boot log path. cloud-init also captures stdout/stderr, but an explicit log file +# under /var/log is the artifact the sec 9.5 finalizer would upload to the +# runner-logs Blob container before teardown. +BOOT_LOG="/var/log/geos-runner-bootstrap.log" + +# The provisioning control plane cannot infer a terminal custom-data failure +# from GitHub's pre-created offline JIT record. This root-owned, atomically +# replaced status file exposes only controlled state, phase, exit code, and UTC +# time. It never contains a command, argument, environment value, custom-data +# payload, or credential. Azure Run Command reads it as data (never `source`s +# it) while the VM is still isolated from repository workload code. +BOOTSTRAP_STATUS_FILE="/run/geos-runner-bootstrap.status" +BOOTSTRAP_PHASE="initializing" + +# AzCopy is ensured on the selected host image at boot because artifact and +# baseline transfers use it directly from the runner host. The version and +# release tarball hashes are fixed here so the root bootstrap never unpacks a +# mutable redirect target. When this is bumped, update both architecture hashes +# from the official release metadata and keep the live drill on the cheap SKU +# until the first boot path has been revalidated. +AZCOPY_VERSION="10.32.4" +AZCOPY_LINUX_AMD64_SHA256="8f859a0dbbc117660c249fb3569694fc8a0f33b68701f5b2b92ccc001ee50784" +AZCOPY_LINUX_ARM64_SHA256="c614777841277ab2c53eecc9ecca5704fd697375c2ffaf4a407058891f00f673" + +log() { + # Timestamped so the serial-console/boot-diagnostics view (plan sec 9.5) shows how + # long each phase took on a cold boot -- the latency the custom-image bake is + # meant to cut. + printf '%s [geos-bootstrap] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "${BOOT_LOG}" +} + +die() { + printf '%s [geos-bootstrap] ERROR: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "${BOOT_LOG}" >&2 + exit 1 +} + +# download_with_retries: fetch one public bootstrap artifact with bounded retry. +# +# The runner starts from a clean Marketplace VM, so Docker/Azure signing keys and +# the pinned GitHub release assets cannot come from a warm local cache. Treat +# transport failures and transient HTTP responses as retryable, but keep each +# request and the complete retry sequence bounded so provisioning cannot leave a +# paid VM waiting forever. Callers still SHA-verify executable release archives +# before unpacking them. +download_with_retries() { + local url="$1" + local output="$2" + local description="$3" + local attempt + + for attempt in 1 2 3 4 5; do + rm -f "${output}" + log "downloading ${description} (attempt ${attempt}/5)" + if curl --fail --silent --show-error --location \ + --connect-timeout 20 \ + --max-time 180 \ + --output "${output}" \ + "${url}"; then + return 0 + fi + + if (( attempt < 5 )); then + log "download of ${description} failed; retrying after bounded backoff" + sleep $(( attempt * 5 )) + fi + done + + die "failed to download ${description} after 5 attempts" +} + +# write_bootstrap_status: atomically publish one sanitized lifecycle record. +# +# Entry state: callers supply only controlled literal states/phases and the +# shell's numeric exit status. Exit state: readers see either the complete old +# record or the complete new record; a partial write can never be mistaken for +# a valid running bootstrap. The explicit allowlists keep future callers from +# turning this diagnostic channel into an accidental command/environment log. +write_bootstrap_status() { + local state="$1" + local phase="$2" + local exit_code="$3" + local status_tmp + + case "${state}" in + running|failed|exited) + ;; + *) + return 1 + ;; + esac + [[ "${phase}" =~ ^[a-z0-9-]+$ ]] || return 1 + [[ "${exit_code}" =~ ^[0-9]+$ ]] || return 1 + + status_tmp="$(mktemp "${BOOTSTRAP_STATUS_FILE}.XXXXXX")" + { + echo "version=1" + echo "state=${state}" + echo "phase=${phase}" + echo "exit_code=${exit_code}" + echo "updated_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } > "${status_tmp}" + chmod 0644 "${status_tmp}" + mv -f "${status_tmp}" "${BOOTSTRAP_STATUS_FILE}" +} + +# set_bootstrap_phase: expose the next deterministic bootstrap operation. +# +# The phase is written before its operation starts, so an EXIT caused by that +# operation retains the exact safe phase name. The corresponding function logs +# its ordinary progress separately; this record exists for machine-readable +# control-plane failure detection rather than verbose tracing. +set_bootstrap_phase() { + BOOTSTRAP_PHASE="$1" + write_bootstrap_status running "${BOOTSTRAP_PHASE}" 0 +} + +# record_bootstrap_exit: preserve the terminal result without leaking context. +# +# Bash passes the status of the command that ended main through `$?`. Disable +# the trap before doing any reporting so a diagnostic failure cannot recurse. +# A zero exit means the foreground JIT runner has already exited; while the +# provisioning health check is still waiting, both `failed` and `exited` are +# terminal because no agent remains that could become online. +record_bootstrap_exit() { + local exit_code="$?" + local terminal_state="failed" + + trap - EXIT + set +e + if (( exit_code == 0 )); then + terminal_state="exited" + fi + if ! write_bootstrap_status "${terminal_state}" "${BOOTSTRAP_PHASE}" "${exit_code}"; then + printf '%s [geos-bootstrap] ERROR: unable to publish terminal bootstrap status\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" | tee -a "${BOOT_LOG}" >&2 + fi + printf '%s [geos-bootstrap] terminal state=%s phase=%s exit_code=%s\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + "${terminal_state}" "${BOOTSTRAP_PHASE}" "${exit_code}" | + tee -a "${BOOT_LOG}" + exit "${exit_code}" +} + +# require_placeholders: assert every templated value was substituted. +# +# Entry state: the provision job has (or should have) replaced every @@...@@ +# token. If any token survives, templating failed and proceeding would create a +# VM that bills but never becomes a usable runner (e.g. an unsubstituted +# jitconfig cannot register), leaving the post-boot health check to time out and +# the cleanup/janitor to reclaim it. Failing loudly here makes the misconfig +# obvious in the boot log instead of manifesting as a mysterious health-check +# timeout. We grep for the literal sentinel pattern rather than checking each +# variable so a newly added placeholder cannot be forgotten. +require_placeholders() { + local v + for v in \ + "${RUNNER_ENCODED_JITCONFIG}" \ + "${RUNNER_VERSION}" \ + "${RUNNER_TARBALL_SHA256}" \ + "${ACR_NAME}" \ + "${RUNNER_VM_UAMI_CLIENT_ID}" \ + "${RUNNER_STORAGE_PROFILE}" \ + "${STORAGE_HELPER_GZIP_BASE64}" \ + "${STORAGE_HELPER_SHA256}" \ + "${HOST_VALIDATOR_GZIP_BASE64}" \ + "${HOST_VALIDATOR_SHA256}" \ + "${CONTAINER_VALIDATOR_GZIP_BASE64}" \ + "${CONTAINER_VALIDATOR_SHA256}" \ + "${PARALLEL_HDF5_SOURCE_GZIP_BASE64}" \ + "${PARALLEL_HDF5_SOURCE_SHA256}"; do + case "${v}" in + *@@*@@*) + die "unsubstituted template placeholder detected ('${v}'); the provision job did not template runner-bootstrap.sh correctly" + ;; + esac + done + + case "${RUNNER_STORAGE_PROFILE}" in + none|hbv4-nvme-raid0) + ;; + *) + die "unsupported RUNNER_STORAGE_PROFILE '${RUNNER_STORAGE_PROFILE}'" + ;; + esac + local digest + for digest in \ + "${STORAGE_HELPER_SHA256}" \ + "${HOST_VALIDATOR_SHA256}" \ + "${CONTAINER_VALIDATOR_SHA256}" \ + "${PARALLEL_HDF5_SOURCE_SHA256}"; do + [[ "${digest}" =~ ^[0-9a-f]{64}$ ]] \ + || die "embedded payload SHA-256 is not a lowercase 64-character digest" + done +} + +# install_base_packages: install the thin host tooling layer. +# +# The GEOS build runs inside the container image (which carries the compiler/TPL +# toolchain), so every host needs the ordinary runner/network tools. Only the +# HBv4 profile adds mdadm, XFS, NVMe, and mount-inspection packages; profile +# "none" retains the exact historical package set and performs no storage-helper +# decode, install, discovery, or mutation. Idempotent: apt-get install is safe to +# re-run, and on a pre-baked image these packages are already present. +install_base_packages() { + log "installing base host packages" + export DEBIAN_FRONTEND=noninteractive + local -a packages=( + curl ca-certificates jq gh git git-lfs tar gzip apt-transport-https lsb-release gnupg + ) + if [[ "${RUNNER_STORAGE_PROFILE}" == "hbv4-nvme-raid0" ]]; then + packages+=(mdadm xfsprogs nvme-cli util-linux) + fi + apt-get update -y + apt-get install -y --no-install-recommends "${packages[@]}" + git lfs install --system || true +} + +# install_storage_helper: reconstruct and authenticate the reviewed disk helper. +# +# The provision job embeds a gzip/base64 representation because Azure accepts +# only one bounded custom-data payload. Decode to a private temporary file, +# verify the uncompressed bytes against the separately embedded SHA-256, and +# install only those authenticated bytes. main calls this only for the HBv4 +# profile; profile "none" still validates the inert placeholder and digest shape +# but preserves its historical host package and filesystem behavior. +install_storage_helper() { + local tmp computed + tmp="$(mktemp)" + if ! printf '%s' "${STORAGE_HELPER_GZIP_BASE64}" \ + | base64 --decode \ + | gzip -dc > "${tmp}"; then + rm -f "${tmp}" + die "unable to decode the embedded HBv4 storage helper" + fi + + computed="$(sha256sum "${tmp}" | awk '{print $1}')" + if [[ "${computed}" != "${STORAGE_HELPER_SHA256}" ]]; then + rm -f "${tmp}" + die "storage helper SHA-256 mismatch: expected ${STORAGE_HELPER_SHA256}, got ${computed}" + fi + install -m 0755 "${tmp}" "${STORAGE_HELPER_PATH}" + rm -f "${tmp}" + log "embedded storage helper SHA-256 verified and installed" +} + +# Install one trusted, independently hashed source-controlled payload without +# ever evaluating its contents in the control-plane renderer. +install_embedded_file() { + local encoded="$1" + local expected_sha256="$2" + local destination="$3" + local mode="$4" + local tmp computed + + tmp="$(mktemp)" + if ! printf '%s' "${encoded}" | base64 --decode | gzip -dc > "${tmp}"; then + rm -f "${tmp}" + die "unable to decode trusted payload for ${destination}" + fi + computed="$(sha256sum "${tmp}" | awk '{print $1}')" + if [[ "${computed}" != "${expected_sha256}" ]]; then + rm -f "${tmp}" + die "trusted payload SHA-256 mismatch for ${destination}" + fi + install -m "${mode}" "${tmp}" "${destination}" + rm -f "${tmp}" +} + +install_trusted_validation() { + install -d -m 0755 "${TRUSTED_VALIDATION_ROOT}" + install_embedded_file \ + "${HOST_VALIDATOR_GZIP_BASE64}" "${HOST_VALIDATOR_SHA256}" \ + "${TRUSTED_VALIDATION_ROOT}/validate-host.sh" 0755 + install_embedded_file \ + "${CONTAINER_VALIDATOR_GZIP_BASE64}" "${CONTAINER_VALIDATOR_SHA256}" \ + "${TRUSTED_VALIDATION_ROOT}/validate-hbv4-tpls" 0755 + install_embedded_file \ + "${PARALLEL_HDF5_SOURCE_GZIP_BASE64}" "${PARALLEL_HDF5_SOURCE_SHA256}" \ + "${TRUSTED_VALIDATION_ROOT}/parallel_hdf5_shared.c" 0644 + log "trusted HBv4 validation payloads verified and installed" +} + +# configure_runner_storage: establish RAID-backed runner and workload state. +# +# Profile "none" deliberately returns without changing the historical paths or +# Docker configuration. The HBv4 profile invokes the fail-closed disk helper, +# then places the GitHub runner home (including its relative _work directory) on +# the XFS RAID mount. It also creates one sticky, world-writable host directory +# that the profile-specific workload wiring binds over container /tmp. Docker's +# images, logs, and writable layers are also placed on the same RAID mount. +configure_runner_storage() { + if [[ "${RUNNER_STORAGE_PROFILE}" == "none" ]]; then + log "runner storage profile none; preserving historical OS-disk paths" + return 0 + fi + + log "configuring HBv4 local NVMe RAID-0 runner storage" + # The helper's accepted/rejected device observations are credential-free and + # are needed to diagnose a failed discovery after cleanup removes the VM. + # pipefail preserves the helper's nonzero status while tee copies the exact + # output to both cloud-init and the reviewed bootstrap log. + if ! "${STORAGE_HELPER_PATH}" 2>&1 | tee -a "${BOOT_LOG}"; then + die "HBv4 local-NVMe storage helper failed" + fi + if ! mountpoint -q "${HBV4_MOUNT_ROOT}"; then + die "storage helper returned without mounting ${HBV4_MOUNT_ROOT}" + fi + + RUNNER_HOME="${HBV4_MOUNT_ROOT}/actions-runner" + install -m 0755 -d "${RUNNER_HOME}" + # The sticky bit gives each container process a conventional /tmp contract + # while preventing one user from removing another user's entries. `--mount` + # in the workload fails if this source disappears, and the final validation + # below independently proves that the directory still resolves to the RAID. + install -o root -g root -m 1777 -d "${HBV4_CONTAINER_TMP}" +} + +# install_docker: install Docker Engine and start the daemon. +# +# The workload job's build is `docker run ...` ON THE HOST (plan sec 2/sec 3), so a +# working Docker daemon is a hard prerequisite before the runner accepts the job. +# We install from Docker's official apt repository (stable channel) pinned to the +# detected Ubuntu codename. The runner user is added to the docker group so the +# agent can launch containers without per-call sudo. Idempotent on a pre-baked +# image (install + enable are no-ops if already satisfied). +install_docker() { + if ! command -v docker >/dev/null 2>&1; then + log "installing Docker Engine from Docker apt repo" + export DEBIAN_FRONTEND=noninteractive + local docker_key + install -m 0755 -d /etc/apt/keyrings + docker_key="$(mktemp)" + download_with_retries \ + "https://download.docker.com/linux/ubuntu/gpg" \ + "${docker_key}" \ + "Docker repository signing key" + gpg --dearmor -o /etc/apt/keyrings/docker.gpg < "${docker_key}" + rm -f "${docker_key}" + chmod a+r /etc/apt/keyrings/docker.gpg + local codename + # shellcheck source=/dev/null + codename="$(. /etc/os-release && echo "${VERSION_CODENAME}")" + printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu %s stable\n' \ + "$(dpkg --print-architecture)" "${codename}" > /etc/apt/sources.list.d/docker.list + apt-get update -y + apt-get install -y --no-install-recommends \ + docker-ce docker-ce-cli containerd.io docker-buildx-plugin + else + log "docker already present" + fi + + if [[ "${RUNNER_STORAGE_PROFILE}" == "hbv4-nvme-raid0" ]]; then + local daemon_tmp + install -m 0711 -d "${HBV4_MOUNT_ROOT}/docker" + install -m 0755 -d /etc/docker + daemon_tmp="$(mktemp)" + if [[ -s /etc/docker/daemon.json ]]; then + jq -e 'type == "object"' /etc/docker/daemon.json >/dev/null \ + || die "/etc/docker/daemon.json is not a JSON object" + jq --arg root "${HBV4_MOUNT_ROOT}/docker" '. + {"data-root": $root}' \ + /etc/docker/daemon.json > "${daemon_tmp}" + else + jq -n --arg root "${HBV4_MOUNT_ROOT}/docker" '{"data-root": $root}' > "${daemon_tmp}" + fi + install -m 0644 "${daemon_tmp}" /etc/docker/daemon.json + rm -f "${daemon_tmp}" + fi + systemctl enable --now docker + systemctl restart docker +} + +# ensure_azure_cli: guarantee `az` and `azcopy` are available. +# +# `az` is needed for the identity + ACR login below; `azcopy` is the Blob +# transfer tool the artifact/baseline provider commands invoke on the runner +# (plan sec 8.3/sec 9.3). Some Ubuntu images ship `az`; otherwise we add Microsoft's +# signed apt repository explicitly instead of piping a downloaded installer into +# root. AzCopy is fetched from a versioned GitHub release URL and SHA-verified +# before unpacking. The Azure CLI repository operations tolerate transient +# downloads through apt's native acquire retries. Package installation also +# waits for apt's dpkg lock; repository-index updates use a separate lists lock, +# so only that exact contention error receives a bounded command retry. Other +# update failures remain immediately fatal because without `az` the ACR login +# cannot happen and the in-job docker pull would fail. +ensure_azure_cli() { + if ! command -v az >/dev/null 2>&1; then + log "installing Azure CLI from Microsoft signed apt repo" + export DEBIAN_FRONTEND=noninteractive + local apt_output apt_status attempt codename dpkg_arch microsoft_key + # Callers use different reviewed Ubuntu Marketplace images, so deriving the + # codename preserves the signed repository boundary across both the manual + # proof and automated HBv4 routes. The package version is intentionally left + # to the repo so security updates flow until the custom-image bake pins host + # tooling. + # shellcheck source=/dev/null + codename="$(. /etc/os-release && echo "${VERSION_CODENAME}")" + dpkg_arch="$(dpkg --print-architecture)" + install -m 0755 -d /etc/apt/keyrings + microsoft_key="$(mktemp)" + download_with_retries \ + "https://packages.microsoft.com/keys/microsoft.asc" \ + "${microsoft_key}" \ + "Microsoft repository signing key" + gpg --dearmor --yes -o /etc/apt/keyrings/microsoft.gpg < "${microsoft_key}" + rm -f "${microsoft_key}" + chmod a+r /etc/apt/keyrings/microsoft.gpg + printf 'deb [arch=%s signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ %s main\n' \ + "${dpkg_arch}" "${codename}" > /etc/apt/sources.list.d/azure-cli.list + # These credential-free markers distinguish repository availability from + # package installation failures in the bounded bootstrap log. + log "updating Azure CLI package repository indexes" + for attempt in 1 2 3 4 5; do + if apt_output="$(apt-get -o Acquire::Retries=3 update -y 2>&1)"; then + printf '%s\n' "${apt_output}" + break + else + apt_status=$? + fi + + # DPkg::Lock::Timeout does not cover apt-get update's lists lock. Retry + # only that recognized transient contention; repository, signature, and + # other deterministic failures return immediately with apt's status. + if (( attempt < 5 )) && + printf '%s\n' "${apt_output}" | + grep -Eq 'Could not get lock /var/lib/apt/lists/lock|Unable to lock directory /var/lib/apt/lists/'; then + log "Azure CLI package repository indexes are locked; retrying after bounded backoff (attempt ${attempt}/5)" + sleep $(( attempt * 10 )) + continue + fi + printf '%s\n' "${apt_output}" >&2 + return "${apt_status}" + done + log "installing Azure CLI package" + apt-get -o Acquire::Retries=3 -o DPkg::Lock::Timeout=120 install -y --no-install-recommends azure-cli + else + log "Azure CLI already present" + fi + + if ! command -v azcopy >/dev/null 2>&1; then + log "installing azcopy v${AZCOPY_VERSION}" + local azcopy_arch azcopy_sha azcopy_url computed tmp + case "$(dpkg --print-architecture)" in + amd64) + azcopy_arch="amd64" + azcopy_sha="${AZCOPY_LINUX_AMD64_SHA256}" + ;; + arm64) + azcopy_arch="arm64" + azcopy_sha="${AZCOPY_LINUX_ARM64_SHA256}" + ;; + *) + die "unsupported architecture for azcopy: $(dpkg --print-architecture)" + ;; + esac + azcopy_url="https://github.com/Azure/azure-storage-azcopy/releases/download/v${AZCOPY_VERSION}/azcopy_linux_${azcopy_arch}_${AZCOPY_VERSION}.tar.gz" + tmp="$(mktemp -d)" + download_with_retries \ + "${azcopy_url}" \ + "${tmp}/azcopy.tar.gz" \ + "AzCopy release tarball" + computed="$(sha256sum "${tmp}/azcopy.tar.gz" | awk '{print $1}')" + if [[ "${computed}" != "${azcopy_sha}" ]]; then + die "azcopy tarball SHA-256 mismatch: expected ${azcopy_sha}, got ${computed} (refusing to unpack an unverified azcopy)" + fi + log "azcopy tarball SHA-256 verified" + tar -xzf "${tmp}/azcopy.tar.gz" -C "${tmp}" + install -m 0755 "$(find "${tmp}" -name azcopy -type f | head -1)" /usr/local/bin/azcopy + rm -rf "${tmp}" + else + log "azcopy already present" + fi +} + +# azure_login_and_acr_login: authenticate the VM to Azure and ACR via the UAMI. +# +# Entry state: the VM was created with --assign-identity , so +# the UAMI is reachable through IMDS. Running AS ${RUNNER_USER} (see body), we log +# in with that identity and then run `az acr login`, which writes Docker +# credentials into that user's ~/.docker/config.json so the later in-job +# `docker pull /geosx/...:` -- which also runs as ${RUNNER_USER} -- is +# authenticated by the UAMI's AcrPull role (plan sec 8.1). Exit state: ${RUNNER_USER} +# can pull from the foundation ACR with no stored registry secret. +# +# --client-id is REQUIRED (plan sec 8.3): id-geos-runner-vm is a +# user-assigned identity, so a bare `az login --identity` would try the +# system-assigned identity, which this VM does not have, and fail. RBAC +# propagation can briefly 403 right after VM create, so we retry with backoff +# rather than treating the first failure as fatal (plan sec 5.4). +azure_login_and_acr_login() { + # Run AS ${RUNNER_USER}, not root. az acr login writes Docker credentials into the + # INVOKING user's ~/.docker/config.json, and the workload's + # `docker run /geosx/...` executes as ${RUNNER_USER} (run.sh runs as that + # user, HOME=/home/${RUNNER_USER}). Doing this as root authenticates root while + # the real pull (as ${RUNNER_USER}) gets "authentication required" and az reports + # docker exit 125. ${RUNNER_USER} must already exist (install_runner_agent creates + # it) and be in the docker group; IMDS is reachable by any user for --identity. + log "az login via user-assigned managed identity (client-id ${RUNNER_VM_UAMI_CLIENT_ID}) as ${RUNNER_USER}" + local attempt + for attempt in 1 2 3 4 5; do + if sudo -u "${RUNNER_USER}" -H az login --identity --client-id "${RUNNER_VM_UAMI_CLIENT_ID}" >/dev/null 2>&1; then + break + fi + log "az login attempt ${attempt} failed (possible RBAC propagation delay); retrying" + sleep $(( attempt * 10 )) + if [[ "${attempt}" == "5" ]]; then + die "az login --identity failed after retries; cannot authenticate ACR pull" + fi + done + + log "az acr login --name ${ACR_NAME} as ${RUNNER_USER}" + for attempt in 1 2 3 4 5; do + if sudo -u "${RUNNER_USER}" -H az acr login --name "${ACR_NAME}" >/dev/null 2>&1; then + return 0 + fi + log "az acr login attempt ${attempt} failed; retrying" + sleep $(( attempt * 10 )) + done + die "az acr login failed after retries; in-job docker pull from ACR would fail" +} + +# install_runner_agent: download, SHA-256 VERIFY, and unpack the pinned runner. +# +# SHA-pin gotcha (plan deliverable): pinning only the version is not enough -- a +# tarball fetched over the network must be integrity-checked before it is +# unpacked and executed as the runner agent. We compute the SHA-256 of the +# downloaded tarball and compare it to the expected value templated in from the +# release metadata; a mismatch aborts (die) BEFORE anything from the tarball runs, +# so a corrupted or substituted download can never execute on the VM. The agent +# runs as a dedicated non-root user (the agent refuses to configure as root); we +# create that user, unpack into its home, and let the bundled installdependencies +# script pull the agent's own OS libs. +install_runner_agent() { + log "installing GitHub Actions runner agent v${RUNNER_VERSION}" + + if ! id -u "${RUNNER_USER}" >/dev/null 2>&1; then + useradd --create-home --shell /bin/bash "${RUNNER_USER}" + fi + usermod -aG docker "${RUNNER_USER}" || true + + mkdir -p "${RUNNER_HOME}" + chown "${RUNNER_USER}:${RUNNER_USER}" "${RUNNER_HOME}" + + local arch tarball url tmp computed + # The runner ships per-arch tarballs; map dpkg arch to the runner's naming. + case "$(dpkg --print-architecture)" in + amd64) arch="x64" ;; + arm64) arch="arm64" ;; + *) die "unsupported architecture for runner agent: $(dpkg --print-architecture)" ;; + esac + tarball="actions-runner-linux-${arch}-${RUNNER_VERSION}.tar.gz" + url="https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/${tarball}" + + tmp="$(mktemp -d)" + download_with_retries \ + "${url}" \ + "${tmp}/${tarball}" \ + "GitHub Actions runner tarball" + + # Integrity gate: verify BEFORE unpacking. We only enforce the pin for the + # x64 tarball whose SHA was templated in; if a different arch is ever used the + # provision job must template the matching SHA, otherwise we refuse to proceed. + computed="$(sha256sum "${tmp}/${tarball}" | awk '{print $1}')" + if [[ "${computed}" != "${RUNNER_TARBALL_SHA256}" ]]; then + die "runner tarball SHA-256 mismatch: expected ${RUNNER_TARBALL_SHA256}, got ${computed} (refusing to unpack an unverified runner)" + fi + log "runner tarball SHA-256 verified" + + tar -xzf "${tmp}/${tarball}" -C "${RUNNER_HOME}" + chown -R "${RUNNER_USER}:${RUNNER_USER}" "${RUNNER_HOME}" + rm -rf "${tmp}" + + # The agent's bundled dependency installer needs root; run it from the unpacked + # tree. It is idempotent and a no-op on a pre-baked image. + if [[ -x "${RUNNER_HOME}/bin/installdependencies.sh" ]]; then + log "installing runner agent OS dependencies" + "${RUNNER_HOME}/bin/installdependencies.sh" + fi +} + +# validate_runner_storage: prove the HBv4 layout before accepting untrusted work. +# +# The disk helper runs before runner installation so runner and workload scratch +# state cannot land on the OS disk. This final boundary rechecks the live +# topology after Docker, Azure tooling, and the runner agent have been +# installed. Any lost mount, incomplete member set, wrong filesystem label, +# undersized device, unwritable runner directory, invalid container scratch +# contract, or unavailable Docker daemon aborts before the JIT runner registers +# and accepts repository-controlled code. +validate_runner_storage() { + if [[ "${RUNNER_STORAGE_PROFILE}" == "none" ]]; then + return 0 + fi + + log "validating HBv4 local NVMe storage before starting JIT runner" + lsblk -e 7 -o NAME,KNAME,PATH,TYPE,SIZE,FSTYPE,LABEL,MOUNTPOINTS,MODEL,SERIAL \ + | tee -a "${BOOT_LOG}" || true + ls -l /dev/disk/by-label 2>&1 | tee -a "${BOOT_LOG}" || true + cat /proc/mdstat | tee -a "${BOOT_LOG}" || true + findmnt --target "${HBV4_MOUNT_ROOT}" | tee -a "${BOOT_LOG}" || true + xfs_info "${HBV4_MOUNT_ROOT}" | tee -a "${BOOT_LOG}" || true + df -hT "${HBV4_MOUNT_ROOT}" | tee -a "${BOOT_LOG}" || true + if ! docker info 2>&1 | tee -a "${BOOT_LOG}"; then + die "Docker daemon is unavailable before runner registration" + fi + + mountpoint -q "${HBV4_MOUNT_ROOT}" \ + || die "${HBV4_MOUNT_ROOT} is not a real mount point" + [[ "$(findmnt -n -o FSTYPE --target "${HBV4_MOUNT_ROOT}")" == "xfs" ]] \ + || die "${HBV4_MOUNT_ROOT} is not mounted as XFS" + case ",$(findmnt -n -o OPTIONS --target "${HBV4_MOUNT_ROOT}")," in + *,noatime,*) + ;; + *) + die "${HBV4_MOUNT_ROOT} is missing the required noatime mount option" + ;; + esac + [[ -b "${HBV4_MD_DEVICE}" ]] \ + || die "expected RAID device ${HBV4_MD_DEVICE} is absent" + + local md_real md_name md_sysfs member_count mount_source + md_real="$(readlink -f "${HBV4_MD_DEVICE}")" + md_name="$(basename "${md_real}")" + md_sysfs="/sys/class/block/${md_name}/md" + [[ -r "${md_sysfs}/level" ]] \ + || die "${HBV4_MD_DEVICE} is not an assembled Linux MD array" + [[ -r "${md_sysfs}/raid_disks" ]] \ + || die "${HBV4_MD_DEVICE} has no readable declared member count" + [[ "$(cat "${md_sysfs}/level")" == "raid0" ]] \ + || die "${HBV4_MD_DEVICE} is not RAID-0" + [[ "$(cat "${md_sysfs}/raid_disks")" == "2" ]] \ + || die "${HBV4_MD_DEVICE} does not declare exactly two RAID members" + # The kernel exposes md/degraded only for levels with redundant members. + # RAID-0 completeness is instead proven by matching its declared geometry to + # the live sysfs slave count before any repository workload can start. + [[ -d "/sys/class/block/${md_name}/slaves" ]] \ + || die "${HBV4_MD_DEVICE} has no readable active member set" + member_count="$(find "/sys/class/block/${md_name}/slaves" -mindepth 1 -maxdepth 1 -type l | wc -l)" + [[ "${member_count}" == "2" ]] \ + || die "${HBV4_MD_DEVICE} has ${member_count} active members, expected 2" + + mount_source="$(findmnt -n -o SOURCE --target "${HBV4_MOUNT_ROOT}")" + [[ "$(readlink -f "${mount_source}")" == "${md_real}" ]] \ + || die "${HBV4_MOUNT_ROOT} is not mounted from ${HBV4_MD_DEVICE}" + [[ "$(blkid -s LABEL -o value "${HBV4_MD_DEVICE}")" == "hbv4-local" ]] \ + || die "${HBV4_MD_DEVICE} does not have XFS label hbv4-local" + + local minimum_bytes total_bytes runner_mount scratch_mount docker_root docker_mount + local scratch_mode runner_probe scratch_probe + minimum_bytes=$(( 3 * 1024 * 1024 * 1024 * 1024 )) + total_bytes="$(df --output=size -B1 "${HBV4_MOUNT_ROOT}" | awk 'NR == 2 {print $1}')" + if ! [[ "${total_bytes}" =~ ^[0-9]+$ ]] || + (( total_bytes < minimum_bytes )); then + die "${HBV4_MOUNT_ROOT} usable capacity ${total_bytes} is below 3 TiB" + fi + + runner_mount="$(findmnt -n -o TARGET -T "${RUNNER_HOME}")" + [[ "${runner_mount}" == "${HBV4_MOUNT_ROOT}" ]] \ + || die "runner home ${RUNNER_HOME} is not backed by ${HBV4_MOUNT_ROOT}" + runner_probe="${RUNNER_HOME}/.bootstrap-write-probe" + sudo -u "${RUNNER_USER}" -H bash -c \ + "printf 'HBv4 bootstrap preflight\\n' > '${runner_probe}'" \ + || die "runner user cannot write to ${RUNNER_HOME}" + rm -f "${runner_probe}" + + [[ -d "${HBV4_CONTAINER_TMP}" ]] \ + || die "container scratch directory ${HBV4_CONTAINER_TMP} is missing" + scratch_mode="$(stat -c '%a' "${HBV4_CONTAINER_TMP}")" + [[ "${scratch_mode}" == "1777" ]] \ + || die "container scratch directory ${HBV4_CONTAINER_TMP} has mode ${scratch_mode}, expected 1777" + scratch_mount="$(findmnt -n -o TARGET -T "${HBV4_CONTAINER_TMP}")" + [[ "${scratch_mount}" == "${HBV4_MOUNT_ROOT}" ]] \ + || die "container scratch directory ${HBV4_CONTAINER_TMP} is not backed by ${HBV4_MOUNT_ROOT}" + scratch_probe="${HBV4_CONTAINER_TMP}/.bootstrap-write-probe" + sudo -u "${RUNNER_USER}" -H bash -c \ + "printf 'HBv4 container scratch preflight\\n' > '${scratch_probe}'" \ + || die "runner user cannot write to container scratch ${HBV4_CONTAINER_TMP}" + rm -f "${scratch_probe}" + + docker_root="$(docker info --format '{{.DockerRootDir}}')" + [[ "${docker_root}" == "${HBV4_MOUNT_ROOT}/docker" ]] \ + || die "Docker root ${docker_root} is not the reviewed HBv4 RAID path" + docker_mount="$(findmnt -n -o TARGET -T "${docker_root}")" + [[ "${docker_mount}" == "${HBV4_MOUNT_ROOT}" ]] \ + || die "Docker root ${docker_root} is not backed by ${HBV4_MOUNT_ROOT}" + + log "HBv4 local NVMe storage validation passed" +} + +# run_runner_ephemeral: start the agent for exactly one job, then exit. +# +# We use the JIT path (`--jitconfig `): the runner registers from the +# encoded config GitHub already minted in the provision job with the unique label +# (the LABEL INVARIANT -- see azure-job.yml), comes online, accepts the ONE job +# routed to that label, runs it, and exits. A JIT runner is inherently +# single-use and is auto-removed from GitHub once it accepts a job; the trio's +# cleanup job still deletes the registration explicitly as a backstop. We run the +# agent in the foreground as the runner user so its exit ends bootstrap; the VM +# is then idle until the trio's cleanup (or the janitor) deletes it. We do NOT +# self-delete the VM from here: that would require giving the runner-VM identity +# VM-delete rights, re-introducing the IMDS escalation risk the plan rejects +# (sec 9.4) -- teardown is the control plane's job. +run_runner_ephemeral() { + local runner_pid + local runner_status + + log "starting ephemeral JIT runner (single job, then exit)" + # Run as the non-root runner user from its home. `run.sh --jitconfig` consumes + # the encoded config on stdin-free argument form; the agent treats a jitconfig + # run as ephemeral automatically (no --once needed, and --once is deprecated). + # + # Start it in the background just long enough to distinguish an immediate + # launch/JIT failure from a live process. Only after that guard succeeds do we + # publish `runner-agent`, which lets the Azure guest monitor hand control back + # to GitHub's authoritative online+label polling. We then wait in the + # foreground for the single-use runner exactly as before. + set_bootstrap_phase runner-launch + sudo -u "${RUNNER_USER}" -H bash -c \ + "cd '${RUNNER_HOME}' && ./run.sh --jitconfig '${RUNNER_ENCODED_JITCONFIG}'" & + runner_pid=$! + sleep 5 + if ! kill -0 "${runner_pid}" 2>/dev/null; then + set +e + wait "${runner_pid}" + runner_status=$? + set -e + die "runner agent exited during its launch guard with status ${runner_status}" + fi + + set_bootstrap_phase runner-agent + set +e + wait "${runner_pid}" + runner_status=$? + set -e + if (( runner_status != 0 )); then + die "runner agent exited with status ${runner_status}" + fi + log "runner agent exited; VM is now idle awaiting trio cleanup / janitor teardown" +} + +main() { + : > "${BOOT_LOG}" || true + trap record_bootstrap_exit EXIT + set_bootstrap_phase initializing + log "GEOS Azure ephemeral runner bootstrap starting" + + set_bootstrap_phase validate-template + require_placeholders + + set_bootstrap_phase install-base-packages + install_base_packages + if [[ "${RUNNER_STORAGE_PROFILE}" == "hbv4-nvme-raid0" ]]; then + set_bootstrap_phase install-storage-helper + install_storage_helper + fi + # Storage setup precedes runner and Docker installation so their mutable + # state cannot silently fall back to the OS disk. + set_bootstrap_phase configure-runner-storage + configure_runner_storage + + set_bootstrap_phase install-trusted-validation + install_trusted_validation + + set_bootstrap_phase install-docker + install_docker + + set_bootstrap_phase install-azure-cli + ensure_azure_cli + # Order matters: install_runner_agent creates ${RUNNER_USER} and adds it to the + # docker group; azure_login_and_acr_login then authenticates ACR AS that user so + # the workload's docker pull (also as ${RUNNER_USER}) finds the credentials. + set_bootstrap_phase install-runner-agent + install_runner_agent + + set_bootstrap_phase azure-login + azure_login_and_acr_login + + set_bootstrap_phase validate-runner-storage + validate_runner_storage + # Re-run the trusted host contract as the exact non-root identity that will + # accept the Actions job. The runner-owned home is the intended writable RAID + # child; the XFS mount root remains protected as root:root. + sudo -u "${RUNNER_USER}" -H "${TRUSTED_VALIDATION_ROOT}/validate-host.sh" \ + --instance-type Standard_HB176rs_v4 \ + --compiler-flags 'target=zen4 -march=native -mtune=native' \ + --writable-path "${RUNNER_HOME}" + + run_runner_ephemeral + log "bootstrap complete" +} + +main "$@" diff --git a/ci/azure/scripts/runner-names.sh b/ci/azure/scripts/runner-names.sh new file mode 100755 index 00000000..4067342e --- /dev/null +++ b/ci/azure/scripts/runner-names.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -euo pipefail + +geos_ci_slugify() { + local raw="${1:-}" + local slug + + slug="$(printf '%s' "${raw}" \ + | tr '[:upper:]' '[:lower:]' \ + | sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//; s/-+/-/g')" + + if [[ -z "${slug}" ]]; then + slug="job" + fi + + printf '%s' "${slug}" +} + +geos_ci_hash() { + if command -v shasum >/dev/null 2>&1; then + printf '%s' "$1" | shasum -a 256 | awk '{ print substr($1, 1, 10) }' + elif command -v sha256sum >/dev/null 2>&1; then + printf '%s' "$1" | sha256sum | awk '{ print substr($1, 1, 10) }' + else + printf '%s' "$1" | cksum | awk '{ print $1 }' + fi +} + +geos_ci_runner_identity() { + local run_id="${1:?github run id is required}" + local run_attempt="${2:?github run attempt is required}" + local runner_instance_key="${3:?runner instance key is required}" + local slug + local base + local suffix + local max_vm_name_length=64 + + if ! [[ "${run_id}" =~ ^[0-9]+$ ]]; then + echo "github run id must be numeric: ${run_id}" >&2 + return 2 + fi + if ! [[ "${run_attempt}" =~ ^[0-9]+$ ]]; then + echo "github run attempt must be numeric: ${run_attempt}" >&2 + return 2 + fi + + slug="$(geos_ci_slugify "${runner_instance_key}")" + # Preserve the TPL workflow's public lifecycle key while using the GEOS + # deterministic identity algorithm and collision-resistant truncation. + base="geos-hbv4-${run_id}-${run_attempt}-${slug}" + + # Azure Linux VM names are capped at 64 characters. The generated GitHub + # runner label and VM name must remain joined by a stable key, so long matrix + # labels are compacted with a deterministic suffix instead of being silently + # truncated to a value that could collide with another matrix row. + if (( ${#base} > max_vm_name_length )); then + suffix="$(geos_ci_hash "${base}")" + slug="${slug:0:$(( max_vm_name_length - 10 - ${#run_id} - ${#run_attempt} - ${#suffix} - 4 ))}" + slug="${slug%-}" + base="geos-hbv4-${run_id}-${run_attempt}-${slug}-${suffix}" + fi + + printf 'runner_name=%s\n' "${base}" + printf 'runner_label=%s\n' "${base}" + printf 'vm_name=%s\n' "${base}" + printf 'log_blob_prefix=github/%s/%s/%s\n' "${run_id}" "${run_attempt}" "${slug}" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + geos_ci_runner_identity "$@" +fi diff --git a/ci/azure/scripts/setup-hbv4-local-nvme.sh b/ci/azure/scripts/setup-hbv4-local-nvme.sh new file mode 100755 index 00000000..04fbb5d9 --- /dev/null +++ b/ci/azure/scripts/setup-hbv4-local-nvme.sh @@ -0,0 +1,523 @@ +#!/usr/bin/env bash +# +# HBv4 local-NVMe setup workflow +# -------------------------------- +# This helper is the only code allowed to turn the two ephemeral HBv4 NVMe +# devices into runner scratch space. Its entry point first discovers and +# validates every candidate without changing block-device state, then either: +# +# * validates and mounts the one already-complete GEOS RAID-0 array; or +# * creates a new RAID-0 array, formats it as XFS, and mounts it. +# +# Every exclusion is deliberately fail-closed. Discovery enumerates kernel +# NVMe namespaces, reads each controller model with `nvme id-ctrl`, and queries +# type and byte size independently so whitespace in an Azure model string can +# never change the field layout. A disk is not considered disposable merely +# because its model matches: it must also be the expected size, be a whole +# unused device, have no signatures or RAID metadata, and be outside all +# protected OS/resource-disk ancestry. Future changes must preserve the +# "validate everything before the first mutation" boundary. + +set -euo pipefail + +readonly EXPECTED_MODEL="Microsoft NVMe Direct Disk" +readonly EXPECTED_SIZE_MIB=1831054 +readonly EXPECTED_SIZE_BYTES=$(( EXPECTED_SIZE_MIB * 1024 * 1024 )) +readonly SIZE_TOLERANCE_BYTES=$(( EXPECTED_SIZE_BYTES / 100 )) +readonly EXPECTED_LABEL="hbv4-local" + +DRY_RUN=false +case "${1:-}" in + "") + ;; + --dry-run) + DRY_RUN=true + ;; + *) + echo "Usage: $0 [--dry-run]" >&2 + exit 2 + ;; +esac + +# Tests exercise destructive-looking branches against regular files and command +# shims. Production ignores all path overrides unless the explicit test-mode +# sentinel is present, so an accidentally inherited individual variable cannot +# redirect storage setup away from the real Azure devices. +if [[ "${GEOS_HBV4_TEST_MODE:-0}" == "1" ]]; then + DEV_ROOT="${GEOS_HBV4_TEST_DEV_ROOT:-/dev}" + BY_ID_DIR="${GEOS_HBV4_TEST_BY_ID_DIR:-/dev/disk/by-id}" + AZURE_DISK_DIR="${GEOS_HBV4_TEST_AZURE_DISK_DIR:-/dev/disk/azure}" + AZURE_LOCAL_BY_SERIAL_DIR="${GEOS_HBV4_TEST_AZURE_LOCAL_BY_SERIAL_DIR:-/dev/disk/azure/local/by-serial}" + SYS_CLASS_BLOCK_DIR="${GEOS_HBV4_TEST_SYS_CLASS_BLOCK_DIR:-/sys/class/block}" + PROC_SWAPS="${GEOS_HBV4_TEST_PROC_SWAPS:-/proc/swaps}" + MD_DEVICE="${GEOS_HBV4_TEST_MD_DEVICE:-/dev/md/hbv4-local}" + MOUNT_POINT="${GEOS_HBV4_TEST_MOUNT_POINT:-/mnt/hbv4-local}" + LOCK_FILE="${GEOS_HBV4_TEST_LOCK_FILE:-/run/lock/geos-hbv4-local-nvme.lock}" +else + DEV_ROOT="/dev" + BY_ID_DIR="/dev/disk/by-id" + AZURE_DISK_DIR="/dev/disk/azure" + AZURE_LOCAL_BY_SERIAL_DIR="/dev/disk/azure/local/by-serial" + SYS_CLASS_BLOCK_DIR="/sys/class/block" + PROC_SWAPS="/proc/swaps" + MD_DEVICE="/dev/md/hbv4-local" + MOUNT_POINT="/mnt/hbv4-local" + LOCK_FILE="/run/lock/geos-hbv4-local-nvme.lock" +fi +readonly DEV_ROOT BY_ID_DIR AZURE_DISK_DIR AZURE_LOCAL_BY_SERIAL_DIR +readonly SYS_CLASS_BLOCK_DIR PROC_SWAPS MD_DEVICE MOUNT_POINT LOCK_FILE + +die() +{ + echo "ERROR: $*" >&2 + exit 1 +} + +[[ "$(id -u)" == "0" ]] || die "HBv4 local-NVMe setup must run as root" + +for command_name in blkid find findmnt flock grep lsblk mdadm mkfs.xfs mount mountpoint nvme readlink xfs_info; do + command -v "${command_name}" >/dev/null 2>&1 || + die "required command is unavailable: ${command_name}" +done + +# The nonblocking process lock prevents two bootstrap invocations from both +# passing discovery before either creates the array. Opening the lock file is +# the only setup-side effect that occurs before block-device validation. +mkdir -p "$(dirname "${LOCK_FILE}")" +exec {LOCK_FD}>"${LOCK_FILE}" +flock -n "${LOCK_FD}" || die "another HBv4 local-NVMe setup process holds ${LOCK_FILE}" + +run_mutation() +{ + if [[ "${DRY_RUN}" == "true" ]]; then + printf 'DRY-RUN:' + printf ' %q' "$@" + printf '\n' + else + "$@" + fi +} + +canonical_path() +{ + readlink -f -- "$1" +} + +# Normalize the fixed-width padding used by `nvme id-ctrl` without interpreting +# any controller-provided characters. The returned model remains the identity +# string used by the exact Azure disk-model allowlist below. +trim_whitespace() +{ + local value="$1" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + printf '%s\n' "${value}" +} + +# Read the NVMe Identify Controller `mn` field from one namespace. Microsoft +# documents this controller query for distinguishing Azure NVMe devices. A +# missing field or failed query is a rejected observation rather than an +# accepted disk; the caller reports that reason before enforcing cardinality. +nvme_controller_model() +{ + local raw_device="$1" + local identify_output + local identify_line + local model + + identify_output="$(nvme id-ctrl "${raw_device}" 2>/dev/null)" || return 1 + while IFS= read -r identify_line; do + if [[ "${identify_line}" =~ ^[[:space:]]*mn[[:space:]]*:[[:space:]]*(.*)$ ]]; then + model="$(trim_whitespace "${BASH_REMATCH[1]}")" + [[ -n "${model}" ]] || return 1 + printf '%s\n' "${model}" + return 0 + fi + done <<<"${identify_output}" + + return 1 +} + +# Select a deterministic stable path when udev exposes one. The kernel name is +# retained as the identity used for topology/sysfs checks, while the stable link +# is used as the mdadm member argument so enumeration order cannot swap members. +stable_member_path() +{ + local raw_device="$1" + local raw_canonical + local stable_dir + local link + local link_canonical + local -a matches=() + + raw_canonical="$(canonical_path "${raw_device}")" || + die "cannot resolve candidate device ${raw_device}" + + # Azure's local-NVMe serial links describe the intended platform identity and + # therefore outrank generic by-id aliases. Each directory is sorted + # independently so adding a lower-priority alias cannot reorder members. + for stable_dir in "${AZURE_LOCAL_BY_SERIAL_DIR}" "${BY_ID_DIR}"; do + matches=() + [[ -d "${stable_dir}" ]] || continue + while IFS= read -r link; do + [[ "${link}" =~ -part[0-9]+$ ]] && continue + link_canonical="$(canonical_path "${link}")" || continue + [[ "${link_canonical}" == "${raw_canonical}" ]] && matches+=("${link}") + done < <(find "${stable_dir}" -maxdepth 1 -type l -print 2>/dev/null | LC_ALL=C sort) + if (( ${#matches[@]} > 0 )); then + printf '%s\n' "${matches[0]}" + return + fi + done + + printf '%s\n' "${raw_device}" +} + +# Return success when candidate_device belongs to the same lsblk ancestry as a +# protected mount source. This catches partitions, dm-crypt/LVM layers, and +# other indirections rather than relying on fragile kernel-name conventions. +device_is_in_source_chain() +{ + local candidate_device="$1" + local source_device="$2" + local chain_device + local candidate_canonical + local chain_output + + candidate_canonical="$(canonical_path "${candidate_device}")" || + die "cannot resolve protected-chain candidate ${candidate_device}" + chain_output="$(lsblk -srnpo NAME -- "${source_device}")" || + die "cannot inspect protected block ancestry for ${source_device}" + + while IFS= read -r chain_device; do + [[ -n "${chain_device}" ]] || continue + if [[ "$(canonical_path "${chain_device}" 2>/dev/null || true)" == "${candidate_canonical}" ]]; then + return 0 + fi + done <<<"${chain_output}" + return 1 +} + +# Collect exact protected mountpoints rather than using findmnt --target, whose +# parent-filesystem fallback would incorrectly classify an ordinary /mnt +# directory as a separate Azure resource disk. +declare -a protected_sources=() +for protected_mount in / /boot /boot/efi /mnt /mnt/resource /mnt/resource_nvme; do + protected_source="" + if protected_source="$(findmnt -rn -o SOURCE --mountpoint "${protected_mount}")"; then + protected_source="${protected_source%%\[*}" + [[ -n "${protected_source}" ]] && + protected_sources+=("${protected_source}") + else + findmnt_status=$? + if [[ "${protected_mount}" == "/" || ${findmnt_status} -gt 1 ]]; then + die "cannot determine protected mount source for ${protected_mount}" + fi + fi +done + +# WALinuxAgent exposes OS and resource-disk identities even when those devices +# are not mounted. Protect their full ancestry explicitly so a transient mount +# failure can never turn an Azure-owned device into a scratch candidate. +for azure_owned_link in "${AZURE_DISK_DIR}/os" "${AZURE_DISK_DIR}/resource"; do + if [[ -e "${azure_owned_link}" ]]; then + protected_sources+=("${azure_owned_link}") + fi +done + +# Enumerate namespace names first, then identify type, byte size, and controller +# model through independent commands. The prior combined lsblk record did not +# classify either live HBv4 disk, and its whitespace-delimited MODEL field made +# the mismatch ambiguous. Reading the controller identity per device removes +# that field-layout dependency. The diagnostics below make every identity +# rejection visible before the exact-two cardinality guard stops setup. +declare -a discovered_records=() +discovery_output="$(lsblk -dnpo NAME)" || + die "cannot enumerate whole block-device names" +while IFS= read -r listed_device; do + listed_device="$(trim_whitespace "${listed_device}")" + [[ -n "${listed_device}" ]] || continue + device_name="$(basename "${listed_device}")" + [[ "${device_name}" =~ ^nvme[0-9]+n[0-9]+$ ]] || continue + + # The enumerated production path is rooted at /dev. Rebuilding it below the + # guarded DEV_ROOT keeps fixtures incapable of addressing host block devices + # while preserving the same kernel basename used by sysfs and topology checks. + raw_device="${DEV_ROOT}/${device_name}" + [[ -b "${raw_device}" || ( "${GEOS_HBV4_TEST_MODE:-0}" == "1" && -e "${raw_device}" ) ]] || + die "enumerated NVMe namespace does not exist as a device: ${raw_device}" + + device_type="$(lsblk -dno TYPE -- "${listed_device}")" || + die "cannot inspect block-device type for ${raw_device}" + device_type="$(trim_whitespace "${device_type}")" + device_size="$(lsblk -dnbo SIZE -- "${listed_device}")" || + die "cannot inspect byte size for ${raw_device}" + device_size="$(trim_whitespace "${device_size}")" + [[ "${device_size}" =~ ^[0-9]+$ ]] || { + printf 'HBv4 NVMe observed: device=%s type=%q model=%q size_bytes=%q decision=rejected reason=%q\n' \ + "${raw_device}" "${device_type}" "" "${device_size}" \ + "byte-size query returned a nonnumeric value" >&2 + die "cannot safely classify ${raw_device} with a nonnumeric byte size" + } + + if ! device_model="$(nvme_controller_model "${raw_device}")"; then + printf 'HBv4 NVMe observed: device=%s type=%q model=%q size_bytes=%s decision=rejected reason=%q\n' \ + "${raw_device}" "${device_type}" "" "${device_size}" \ + "nvme id-ctrl failed or returned no model field" >&2 + continue + fi + + rejection_reason="" + if [[ "${device_type}" != "disk" ]]; then + rejection_reason="lsblk type is not a whole disk" + elif [[ "${device_model}" != "${EXPECTED_MODEL}" ]]; then + rejection_reason="controller model does not match the HBv4 ephemeral-disk allowlist" + else + size_delta=$(( device_size - EXPECTED_SIZE_BYTES )) + (( size_delta < 0 )) && size_delta=$(( -size_delta )) + if (( size_delta > SIZE_TOLERANCE_BYTES )); then + rejection_reason="byte size is outside the one-percent HBv4 tolerance" + fi + fi + + if [[ -n "${rejection_reason}" ]]; then + printf 'HBv4 NVMe observed: device=%s type=%q model=%q size_bytes=%s decision=rejected reason=%q\n' \ + "${raw_device}" "${device_type}" "${device_model}" "${device_size}" \ + "${rejection_reason}" >&2 + continue + fi + + stable_device="$(stable_member_path "${raw_device}")" + discovered_records+=("${stable_device}|${raw_device}|${device_model}|${device_size}") + printf 'HBv4 NVMe observed: device=%s type=%q model=%q size_bytes=%s decision=accepted\n' \ + "${raw_device}" "${device_type}" "${device_model}" "${device_size}" +done <<<"${discovery_output}" + +(( ${#discovered_records[@]} == 2 )) || + die "expected exactly two validated HBv4 ephemeral NVMe devices; found ${#discovered_records[@]}" + +mapfile -t discovered_records < <(printf '%s\n' "${discovered_records[@]}" | LC_ALL=C sort) + +declare -a member_devices=() +declare -a raw_devices=() +for record in "${discovered_records[@]}"; do + IFS='|' read -r member_device raw_device device_model device_size <<<"${record}" + member_devices+=("${member_device}") + raw_devices+=("${raw_device}") + printf 'HBv4 NVMe candidate: device=%s stable=%s model=%q size_bytes=%s\n' \ + "${raw_device}" "${member_device}" "${device_model}" "${device_size}" +done + +[[ "$(canonical_path "${raw_devices[0]}")" != "$(canonical_path "${raw_devices[1]}")" ]] || + die "the two candidate records resolve to the same block device" + +array_exists=false +[[ -e "${MD_DEVICE}" ]] && array_exists=true + +# Validate the destination namespace before any mdadm or mkfs mutation. A lost +# mount can leave files in the underlying OS-disk directory, while a symlink or +# unrelated mount can redirect the eventual mount over state this helper does +# not own. An already-mounted destination is meaningful only when the expected +# array also exists; its exact source/filesystem/options are checked below. +[[ ! -L "${MOUNT_POINT}" ]] || + die "mount point ${MOUNT_POINT} is a symbolic link" +[[ ! -e "${MOUNT_POINT}" || -d "${MOUNT_POINT}" ]] || + die "mount point ${MOUNT_POINT} exists but is not a directory" +if mountpoint -q "${MOUNT_POINT}"; then + [[ "${array_exists}" == "true" ]] || + die "${MOUNT_POINT} is already mounted while ${MD_DEVICE} is absent" +else + mountpoint_status=$? + [[ ${mountpoint_status} -eq 1 ]] || + die "cannot determine whether ${MOUNT_POINT} is mounted" + if [[ -d "${MOUNT_POINT}" ]] && + [[ -n "$(find "${MOUNT_POINT}" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" ]]; then + die "unmounted destination ${MOUNT_POINT} is not empty" + fi +fi + +reject_candidate() +{ + die "candidate ${1} is not pristine ephemeral scratch: ${2}" +} + +# Complete every destructive precondition for both members before deciding +# whether an array should be created. A failure on the second member therefore +# cannot leave a half-created array from an already-accepted first member. +for raw_device in "${raw_devices[@]}"; do + topology_output="$(lsblk -nrpo NAME,TYPE,MOUNTPOINT -- "${raw_device}")" || + die "cannot inspect topology for ${raw_device}" + [[ -n "${topology_output}" ]] || die "lsblk returned no topology for ${raw_device}" + mapfile -t topology_lines <<<"${topology_output}" + read -r topology_name topology_type topology_mount <<<"${topology_lines[0]}" + [[ "$(canonical_path "${topology_name}")" == "$(canonical_path "${raw_device}")" ]] || + reject_candidate "${raw_device}" "lsblk topology begins with a different device" + [[ "${topology_type}" == "disk" ]] || + reject_candidate "${raw_device}" "candidate is not a whole disk" + [[ -z "${topology_mount:-}" ]] || + reject_candidate "${raw_device}" "whole device is mounted at ${topology_mount}" + + if [[ "${array_exists}" == "false" ]]; then + (( ${#topology_lines[@]} == 1 )) || + reject_candidate "${raw_device}" "device has partitions or descendants" + else + # An assembled expected array appears as the sole lsblk descendant of each + # member. Permit only that exact descendant here; its geometry, members, + # filesystem, and mount are independently validated below. + (( ${#topology_lines[@]} == 2 )) || + reject_candidate "${raw_device}" "existing-array member has unexpected descendants" + read -r descendant_name descendant_type descendant_mount <<<"${topology_lines[1]}" + [[ "$(canonical_path "${descendant_name}")" == "$(canonical_path "${MD_DEVICE}")" ]] || + reject_candidate "${raw_device}" "descendant is not the expected GEOS array" + [[ "${descendant_type}" == "raid0" ]] || + reject_candidate "${raw_device}" "expected array descendant is not RAID-0" + [[ -z "${descendant_mount:-}" || "${descendant_mount}" == "${MOUNT_POINT}" ]] || + reject_candidate "${raw_device}" "array descendant is mounted at an unexpected path" + fi + + for protected_source in "${protected_sources[@]}"; do + if device_is_in_source_chain "${raw_device}" "${protected_source}"; then + reject_candidate "${raw_device}" \ + "device overlaps protected OS, boot, EFI, or Azure resource storage" + fi + done + + device_base="$(basename "$(canonical_path "${raw_device}")")" + holders_dir="${SYS_CLASS_BLOCK_DIR}/${device_base}/holders" + if [[ "${array_exists}" == "false" && -d "${holders_dir}" ]] && + compgen -G "${holders_dir}/*" >/dev/null; then + reject_candidate "${raw_device}" "device has block holders" + fi + + if [[ -r "${PROC_SWAPS}" ]]; then + while read -r swap_device _; do + [[ "${swap_device}" == "Filename" || -z "${swap_device}" ]] && continue + if [[ "$(canonical_path "${swap_device}" 2>/dev/null || true)" == \ + "$(canonical_path "${raw_device}")" ]]; then + reject_candidate "${raw_device}" "device is active swap" + fi + done < "${PROC_SWAPS}" + else + die "cannot inspect active swap devices at ${PROC_SWAPS}" + fi +done + +if [[ "${array_exists}" == "true" ]]; then + # An existing array is accepted only if mdadm and sysfs independently agree + # on its geometry and exact active members. RAID-0 has no redundant members, + # so Linux does not expose the redundancy-only md/degraded attribute for this + # level. The declared two-device geometry plus the exact sysfs slave set is + # therefore the fail-closed completeness contract. Member signatures are + # expected here and are validated through the array rather than rejected as + # foreign metadata. + md_level="" + md_devices="" + while IFS='=' read -r detail_key detail_value; do + case "${detail_key}" in + MD_LEVEL) md_level="${detail_value}" ;; + MD_DEVICES) md_devices="${detail_value}" ;; + esac + done < <(mdadm --detail --export "${MD_DEVICE}") + [[ "${md_level}" == "raid0" && "${md_devices}" == "2" ]] || + die "existing ${MD_DEVICE} does not declare a two-device RAID-0 geometry" + + md_base="$(basename "$(canonical_path "${MD_DEVICE}")")" + md_sysfs_dir="${SYS_CLASS_BLOCK_DIR}/${md_base}/md" + [[ -r "${md_sysfs_dir}/level" && + -r "${md_sysfs_dir}/raid_disks" ]] || + die "existing array has no readable sysfs RAID geometry" + read -r sysfs_md_level < "${md_sysfs_dir}/level" + read -r sysfs_raid_disks < "${md_sysfs_dir}/raid_disks" + [[ "${sysfs_md_level}" == "raid0" && + "${sysfs_raid_disks}" == "2" ]] || + die "existing ${MD_DEVICE} does not declare two-device RAID-0 geometry in sysfs" + + slaves_dir="${SYS_CLASS_BLOCK_DIR}/${md_base}/slaves" + [[ -d "${slaves_dir}" ]] || die "existing array has no readable sysfs slave set" + mapfile -t actual_members < <( + for slave_path in "${slaves_dir}"/*; do + [[ -e "${slave_path}" ]] && basename "${slave_path}" + done | LC_ALL=C sort + ) + mapfile -t expected_members < <( + for raw_device in "${raw_devices[@]}"; do + basename "$(canonical_path "${raw_device}")" + done | LC_ALL=C sort + ) + [[ "${actual_members[*]}" == "${expected_members[*]}" ]] || + die "existing array members do not exactly match the two validated HBv4 devices" + + # Each member must be held only by this array. This closes the gap left by + # allowing the expected md descendant during generic topology validation. + for raw_device in "${raw_devices[@]}"; do + device_base="$(basename "$(canonical_path "${raw_device}")")" + member_holders_dir="${SYS_CLASS_BLOCK_DIR}/${device_base}/holders" + [[ -d "${member_holders_dir}" ]] || + die "existing array member ${raw_device} has no readable holder set" + mapfile -t actual_holders < <( + for holder_path in "${member_holders_dir}"/*; do + [[ -e "${holder_path}" ]] && basename "${holder_path}" + done | LC_ALL=C sort + ) + [[ "${actual_holders[*]}" == "${md_base}" ]] || + die "existing array member ${raw_device} has an unexpected holder set" + done + + [[ "$(blkid -s TYPE -o value "${MD_DEVICE}")" == "xfs" ]] || + die "existing array filesystem is not XFS" + [[ "$(blkid -s LABEL -o value "${MD_DEVICE}")" == "${EXPECTED_LABEL}" ]] || + die "existing array filesystem label is not ${EXPECTED_LABEL}" + xfs_geometry="$(xfs_info "${MD_DEVICE}")" || + die "cannot inspect existing XFS geometry" + grep -Eq '(^|[[:space:]])ftype=1([[:space:]]|$)' <<<"${xfs_geometry}" || + die "existing XFS filesystem does not have directory ftype=1" +else + # No array may be created if either member already carries any filesystem or + # RAID signature. These probes intentionally distinguish "nothing found" + # from an operational inspection error. + for raw_device in "${raw_devices[@]}"; do + if blkid -p "${raw_device}" >/dev/null 2>&1; then + reject_candidate "${raw_device}" "device contains a filesystem or partition signature" + else + probe_status=$? + [[ ${probe_status} -eq 2 ]] || + reject_candidate "${raw_device}" "filesystem-signature inspection failed" + fi + + if mdadm --examine "${raw_device}" >/dev/null 2>&1; then + reject_candidate "${raw_device}" "device contains stale or foreign RAID metadata" + else + examine_status=$? + [[ ${examine_status} -eq 1 ]] || + reject_candidate "${raw_device}" "RAID-metadata inspection failed" + fi + done + + run_mutation mkdir -p "$(dirname "${MD_DEVICE}")" + run_mutation mdadm --create "${MD_DEVICE}" --run --level=0 --raid-devices=2 \ + --metadata=1.2 "${member_devices[@]}" + run_mutation mkfs.xfs -f -L "${EXPECTED_LABEL}" -n ftype=1 "${MD_DEVICE}" +fi + +if mountpoint -q "${MOUNT_POINT}"; then + mounted_record="$(findmnt -rn -o SOURCE,FSTYPE,OPTIONS --mountpoint "${MOUNT_POINT}")" || + die "cannot inspect existing mount at ${MOUNT_POINT}" + read -r mounted_source mounted_type mounted_options <<<"${mounted_record}" + [[ "$(canonical_path "${mounted_source}")" == "$(canonical_path "${MD_DEVICE}")" ]] || + die "${MOUNT_POINT} is mounted from an unexpected source" + [[ "${mounted_type}" == "xfs" ]] || die "${MOUNT_POINT} is not mounted as XFS" + [[ ",${mounted_options}," == *,noatime,* ]] || + die "${MOUNT_POINT} is missing the required noatime option" +else + mountpoint_status=$? + [[ ${mountpoint_status} -eq 1 ]] || + die "cannot determine whether ${MOUNT_POINT} is mounted" + run_mutation mkdir -p "${MOUNT_POINT}" + run_mutation mount -t xfs -o noatime "LABEL=${EXPECTED_LABEL}" "${MOUNT_POINT}" +fi + +if [[ "${DRY_RUN}" == "true" ]]; then + echo "HBv4 local-NVMe dry run completed without changing block-device state." +else + echo "HBv4 local-NVMe RAID-0 is mounted at ${MOUNT_POINT}." +fi diff --git a/ci/azure/scripts/validate-created-hbv4-vm.sh b/ci/azure/scripts/validate-created-hbv4-vm.sh new file mode 100755 index 00000000..636ca01d --- /dev/null +++ b/ci/azure/scripts/validate-created-hbv4-vm.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash + +# Validate the post-create Azure VM model before the paid runner is accepted. +# +# The action passes the unprojected `az vm show` JSON on stdin so this helper +# and its fixtures exercise Azure's real response keys. In particular, the +# Compute schema spells the OS-disk field `diskSizeGB`; validating the raw model +# prevents a misspelled JMESPath projection from silently turning it into null. +# Diagnostics intentionally contain field names only: the complete VM response +# can include deployment metadata that does not belong in Actions logs. + +set -euo pipefail + +if (( $# != 6 )); then + echo "usage: $0 VM_NAME PROVIDER ROLE INSTANCE_KEY RUNNER_LABEL UAMI_ID" >&2 + exit 2 +fi + +vm_name="$1" +provider="$2" +role="$3" +instance_key="$4" +runner_label="$5" +uami_id="$6" +vm_contract="$(cat)" + +if [[ -z "${vm_contract}" ]]; then + echo '::error::Azure returned an empty VM contract.' >&2 + exit 1 +fi + +# Keep the expected machine tuple here, rather than accepting caller-provided +# size/image/location values, so the validation boundary remains fail-closed. +# Azure resource identifiers and platform enum-like fields are case-insensitive; +# lifecycle tag values remain exact because cleanup joins on those strings. +if ! mismatches="$(jq -r \ + --arg name "${vm_name}" \ + --arg provider "${provider}" \ + --arg role "${role}" \ + --arg instance_key "${instance_key}" \ + --arg runner_label "${runner_label}" \ + --arg uami_id "${uami_id}" ' + def lower_string: + if type == "string" then ascii_downcase else "" end; + def nonempty_string: + type == "string" and length > 0; + + (.identity.userAssignedIdentities // null) as $identities + | [ + if .name == $name then empty else "name" end, + if ((.hardwareProfile.vmSize // null) | lower_string) == + "standard_hb176rs_v4" + then empty else "hardwareProfile.vmSize" end, + if ((.location // null) | lower_string) == "southcentralus" + then empty else "location" end, + if .tags.project == "GEOS" then empty else "tags.project" end, + if .tags.purpose == "hbv4-tpl-ci" + then empty else "tags.purpose" end, + if .tags.provider == $provider + then empty else "tags.provider" end, + if .tags.runnerRole == $role + then empty else "tags.runnerRole" end, + if .tags.runnerInstanceKey == $instance_key + then empty else "tags.runnerInstanceKey" end, + if .tags.runnerLabel == $runner_label + then empty else "tags.runnerLabel" end, + if ((.tags.ttlHours // "") | tostring) == "6" + then empty else "tags.ttlHours" end, + if .storageProfile.osDisk.diskSizeGB == 128 + then empty else "storageProfile.osDisk.diskSizeGB" end, + if ((.storageProfile.imageReference.publisher // null) | lower_string) == + "microsoft-dsvm" + then empty else "storageProfile.imageReference.publisher" end, + if ((.storageProfile.imageReference.offer // null) | lower_string) == + "ubuntu-hpc" + then empty else "storageProfile.imageReference.offer" end, + if ((.storageProfile.imageReference.sku // null) | lower_string) == "2404" + then empty else "storageProfile.imageReference.sku" end, + if ((.storageProfile.imageReference.exactVersion // + .storageProfile.imageReference.version // "") | tostring) == + "24.04.2026070201" + then empty else "storageProfile.imageReference.version" end, + if (($identities | type) == "object" and + ([$identities | keys[] | ascii_downcase] | + index($uami_id | ascii_downcase)) != null) + then empty else "identity.userAssignedIdentities" end, + if ((.networkProfile.networkInterfaces[0].id // null) | nonempty_string) + then empty else "networkProfile.networkInterfaces[0].id" end, + if ((.storageProfile.osDisk.managedDisk.id // null) | nonempty_string) + then empty else "storageProfile.osDisk.managedDisk.id" end + ] + | join(", ") + ' <<< "${vm_contract}" 2>/dev/null)"; then + echo '::error::Azure returned malformed VM contract JSON.' >&2 + exit 1 +fi + +if [[ -n "${mismatches}" ]]; then + echo "::error::Azure VM failed fixed HBv4 contract fields: ${mismatches}" >&2 + exit 1 +fi diff --git a/ci/azure/tests/fixtures/hbv4-vm-show.json b/ci/azure/tests/fixtures/hbv4-vm-show.json new file mode 100644 index 00000000..c087df88 --- /dev/null +++ b/ci/azure/tests/fixtures/hbv4-vm-show.json @@ -0,0 +1,47 @@ +{ + "name": "geos-hbv4-123456-1-openmpi-builder", + "location": "southcentralus", + "tags": { + "project": "GEOS", + "purpose": "hbv4-tpl-ci", + "provider": "openmpi", + "runnerRole": "builder", + "runnerInstanceKey": "openmpi-builder", + "runnerLabel": "geos-hbv4-123456-1-openmpi-builder", + "ttlHours": "6" + }, + "identity": { + "type": "UserAssigned", + "userAssignedIdentities": { + "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/rg-geos-ci-foundation-scus/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-geos-runner-vm": {} + } + }, + "hardwareProfile": { + "vmSize": "Standard_HB176rs_v4" + }, + "storageProfile": { + "imageReference": { + "publisher": "microsoft-dsvm", + "offer": "ubuntu-hpc", + "sku": "2404", + "version": "24.04.2026070201", + "exactVersion": "24.04.2026070201" + }, + "osDisk": { + "managedDisk": { + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-geos-ci-runners-scus/providers/Microsoft.Compute/disks/hbv4-os-disk" + }, + "diskSizeGB": 128 + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-geos-ci-runners-scus/providers/Microsoft.Network/networkInterfaces/hbv4-nic" + } + ] + }, + "osProfile": { + "customData": "DO_NOT_LOG_VM_METADATA" + } +} diff --git a/ci/azure/tests/test-cleanup-runner-registration.sh b/ci/azure/tests/test-cleanup-runner-registration.sh new file mode 100755 index 00000000..094c46fa --- /dev/null +++ b/ci/azure/tests/test-cleanup-runner-registration.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash + +# Credential-free regression fixtures for the GitHub-registration half of the +# cleanup composite. The test executes the production action's embedded shell +# verbatim with a PATH-injected `gh` shim; no Azure or GitHub resource is used. + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +action="${repo_root}/ci/azure/actions/cleanup-runner/action.yml" +test_root="$(mktemp -d)" +trap 'rm -rf "${test_root}"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +ruby -ryaml -e ' + action = YAML.load_file(ARGV.fetch(0)) + step = action.fetch("runs").fetch("steps").find do |item| + item["name"] == "Delete and verify exact runner registration" + end + abort "cleanup registration step is missing" unless step + File.write(ARGV.fetch(1), step.fetch("run")) +' "${action}" "${test_root}/cleanup-registration.sh" +chmod +x "${test_root}/cleanup-registration.sh" + +mkdir -p "${test_root}/bin" +cat > "${test_root}/bin/sleep" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +chmod +x "${test_root}/bin/sleep" + +cat > "${test_root}/bin/gh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +endpoint="${!#}" +state="$(<"${MOCK_STATE_FILE}")" +if [[ "${state}" == present ]]; then + runners="$(<"${MOCK_RUNNERS_FILE}")" +else + runners='[]' +fi + +if [[ " $* " == *' --method DELETE '* ]]; then + case "${MOCK_SCENARIO}" in + delete-404-denied) + printf 'absent\n' > "${MOCK_STATE_FILE}" + echo 'gh: Not Found (HTTP 404)' >&2 + exit 1 + ;; + *) + printf 'absent\n' > "${MOCK_STATE_FILE}" + exit 0 + ;; + esac +fi + +case "${endpoint}" in + *'actions/runners?per_page=100') + count="$(jq -r 'length' <<< "${runners}")" + jq -cn --argjson count "${count}" --argjson runners "${runners}" \ + '[{total_count:$count,runners:$runners}]' + ;; + *'actions/runners?per_page=1') + if [[ "${MOCK_SCENARIO}" == delete-404-denied ]]; then + echo 'gh: Resource not accessible by integration (HTTP 403)' >&2 + exit 1 + fi + count="$(jq -r 'length' <<< "${runners}")" + jq -cn --argjson count "${count}" --argjson runners "${runners}" \ + '{total_count:$count,runners:($runners[0:1])}' + ;; + *'/actions/runners/'*) + runner_id="${endpoint##*/}" + match="$(jq -c --argjson id "${runner_id}" '[.[] | select(.id == $id)]' <<< "${runners}")" + if (( $(jq -r 'length' <<< "${match}") == 1 )); then + jq -c '.[0]' <<< "${match}" + exit 0 + fi + echo 'gh: Not Found (HTTP 404)' >&2 + exit 1 + ;; + *) + echo "unexpected gh invocation: $*" >&2 + exit 99 + ;; +esac +EOF +chmod +x "${test_root}/bin/gh" + +canonical_label='geos-hbv4-31555791403-1-openmpi-builder' + +runner_json() { + local id="$1" name="$2" label="$3" + jq -cn \ + --argjson id "${id}" \ + --arg name "${name}" \ + --arg label "${label}" \ + '{id:$id,name:$name,status:"offline",busy:false,labels:[{id:0,name:$label,type:"read-only"}]}' +} + +run_fixture() { + local scenario="$1" runners="$2" provision_id="${3:-}" + local case_dir="${test_root}/${scenario}" + mkdir -p "${case_dir}" + printf '%s\n' "${runners}" > "${case_dir}/runners.json" + printf 'present\n' > "${case_dir}/state" + PATH="${test_root}/bin:${PATH}" \ + GH_TOKEN='fixture-runner-admin-token' \ + GITHUB_REPOSITORY='earthflow-sim/thirdPartyLibs' \ + GITHUB_RUN_ID='31555791403' \ + GITHUB_RUN_ATTEMPT='1' \ + PROVIDER='openmpi' \ + RUNNER_ROLE='builder' \ + PROVISION_RUNNER_ID="${provision_id}" \ + MOCK_SCENARIO="${scenario}" \ + MOCK_RUNNERS_FILE="${case_dir}/runners.json" \ + MOCK_STATE_FILE="${case_dir}/state" \ + bash "${test_root}/cleanup-registration.sh" \ + >"${case_dir}/stdout" 2>"${case_dir}/stderr" +} + +legacy_runner="$(runner_json 22 'GitHub Actions 1000003390' "${canonical_label}")" +run_fixture legacy-protected-name "[${legacy_runner}]" +[[ "$(<"${test_root}/legacy-protected-name/state")" == absent ]] || + fail 'protected-name regression was not deleted by exact lifecycle label' + +canonical_runner="$(runner_json 41 "${canonical_label}" "${canonical_label}")" +run_fixture canonical-name "[${canonical_runner}]" 41 +[[ "$(<"${test_root}/canonical-name/state")" == absent ]] || + fail 'canonical exact-ID runner was not deleted' + +duplicate_runner="$(runner_json 42 'GitHub Actions 1000003391' "${canonical_label}")" +if run_fixture duplicate-label "[${legacy_runner},${duplicate_runner}]"; then + fail 'duplicate lifecycle label was not rejected as ambiguous' +fi +[[ "$(<"${test_root}/duplicate-label/state")" == present ]] || + fail 'ambiguous registration was mutated before rejection' + +wrong_label='geos-hbv4-31555791403-1-mpich-builder' +wrong_runner="$(runner_json 43 'GitHub Actions 1000003392' "${wrong_label}")" +if run_fixture mismatched-provision-id "[${wrong_runner}]" 43; then + fail 'supplied ID without the requested provider label was accepted' +fi +[[ "$(<"${test_root}/mismatched-provision-id/state")" == present ]] || + fail 'mismatched supplied ID was mutated before rejection' + +missing_label="$(jq -cn --arg name "${canonical_label}" \ + '{id:44,name:$name,status:"offline",busy:false,labels:[]}')" +if run_fixture canonical-name-missing-label "[${missing_label}]"; then + fail 'canonical name without its matching lifecycle label was accepted' +fi + +malformed_runner='{"id":45,"name":"malformed","status":"offline","busy":false,"labels":null}' +if run_fixture malformed-inventory "[${malformed_runner}]"; then + fail 'malformed runner inventory was accepted' +fi + +if run_fixture delete-404-denied "[${legacy_runner}]"; then + fail 'DELETE 404 was accepted without an independent collection-access proof' +fi + +echo 'cleanup runner registration fixtures passed.' diff --git a/ci/azure/tests/test-created-hbv4-vm-contract.sh b/ci/azure/tests/test-created-hbv4-vm-contract.sh new file mode 100755 index 00000000..304ee391 --- /dev/null +++ b/ci/azure/tests/test-created-hbv4-vm-contract.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# Exercise the exact raw Azure VM response consumed after `az vm create`. +# +# The positive fixture preserves Azure CLI's case-sensitive `diskSizeGB` key. +# Negative mutations protect the fail-closed boundary and ensure diagnostics +# identify only mismatched fields without echoing the VM document. + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +validator="${repo_root}/ci/azure/scripts/validate-created-hbv4-vm.sh" +fixture="${repo_root}/ci/azure/tests/fixtures/hbv4-vm-show.json" +vm_name='geos-hbv4-123456-1-openmpi-builder' +provider='openmpi' +role='builder' +instance_key='openmpi-builder' +runner_label='geos-hbv4-123456-1-openmpi-builder' +uami_id='/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-geos-ci-foundation-scus/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-geos-runner-vm' +tmp_dir="$(mktemp -d)" +trap 'rm -rf "${tmp_dir}"' EXIT + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +validate() +{ + bash "${validator}" \ + "${vm_name}" "${provider}" "${role}" "${instance_key}" \ + "${runner_label}" "${uami_id}" +} + +assert_rejected() +{ + local jq_filter="$1" + local expected_field="$2" + + if jq "${jq_filter}" "${fixture}" | validate \ + >"${tmp_dir}/stdout" 2>"${tmp_dir}/stderr"; then + fail "malformed ${expected_field} contract unexpectedly passed" + fi + grep -Fq "${expected_field}" "${tmp_dir}/stderr" || + fail "${expected_field} rejection omitted its field name" + if grep -Fq 'DO_NOT_LOG_VM_METADATA' "${tmp_dir}/stderr"; then + fail "contract rejection exposed the raw Azure VM response" + fi +} + +validate < "${fixture}" || fail 'canonical Azure VM response was rejected' + +# Regression for the live failure: Azure emits uppercase `GB`. A projection or +# validator using `diskSizeGb` must not make a realistic response appear valid. +assert_rejected \ + '.storageProfile.osDisk |= (del(.diskSizeGB) + {diskSizeGb: 128})' \ + 'storageProfile.osDisk.diskSizeGB' + +assert_rejected \ + '.storageProfile.imageReference.exactVersion = "24.04.unreviewed"' \ + 'storageProfile.imageReference.version' + +if printf '{' | validate >"${tmp_dir}/stdout" 2>"${tmp_dir}/stderr"; then + fail 'malformed JSON unexpectedly passed' +fi +grep -Fq 'malformed VM contract JSON' "${tmp_dir}/stderr" || + fail 'malformed JSON did not produce a classified diagnostic' + +echo 'created HBv4 VM contract fixtures passed.' diff --git a/ci/azure/tests/test-hbv4-build-resilience.sh b/ci/azure/tests/test-hbv4-build-resilience.sh new file mode 100755 index 00000000..eef4a85b --- /dev/null +++ b/ci/azure/tests/test-hbv4-build-resilience.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# Exercise the HBv4 source-fetch retry and MPI-wrapper identity contracts. + +set -euo pipefail + +repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../.." && pwd -P)" +capture_wrapper="${repository_root}/scripts/hbv4/capture-mpi-wrapper-show.sh" +install_from_source="${repository_root}/scripts/hbv4/install-spack-from-source.sh" +collect_evidence="${repository_root}/scripts/hbv4/collect-build-evidence.sh" +container_validator="${repository_root}/scripts/hbv4/validate-hbv4-tpls" +dockerfile="${repository_root}/docker/tpl-ubuntu.Dockerfile" +test_root=$(mktemp -d "${TMPDIR:-/tmp}/hbv4-build-resilience.XXXXXX") +trap 'rm -rf -- "$test_root"' EXIT + +fail() { + printf 'test-hbv4-build-resilience: ERROR: %s\n' "$*" >&2 + exit 1 +} + +assert_contains() { + local file=$1 + local expected=$2 + grep -Fq -- "$expected" "$file" || fail "missing '$expected' in $file" +} + +# Keep the paid HBv4 build on the explicit setup, prefetch, and source-install +# path while leaving the portable Docker build on uberenv's existing path. +assert_contains "$dockerfile" '--setup-and-env-only' +assert_contains "$dockerfile" 'scripts/hbv4/install-spack-from-source.sh' +assert_contains "$dockerfile" "if [ \"\${HBV4_BUILD}\" = 1 ]; then" + +# HDF5's CMake configuration reports true as ON, while older Autotools builds +# report yes. Accept both spellings, reject false values, and require the same +# guard during evidence collection and fresh-runner validation. +parallel_hdf5_pattern='Parallel HDF5:[[:space:]]*(yes|on)([[:space:]]|$)' +assert_contains "$collect_evidence" "parallel_hdf5_pattern='$parallel_hdf5_pattern'" +assert_contains "$container_validator" "parallel_hdf5_pattern='$parallel_hdf5_pattern'" +for enabled_value in 'Parallel HDF5: ON' 'Parallel HDF5: yes'; do + grep -Eiq "$parallel_hdf5_pattern" <<<"$enabled_value" || + fail "parallel-HDF5 pattern rejected enabled value: $enabled_value" +done +if grep -Eiq "$parallel_hdf5_pattern" <<<'Parallel HDF5: OFF'; then + fail 'parallel-HDF5 pattern accepted a disabled value' +fi + +# Spack normalizes the Zen 4 preference to the x86-64-v4 architecture name in +# its lockfile. Require that target for every source-built spec while allowing +# generic x86_64 targets on compilers and other declared externals. +assert_contains "$container_validator" 'source_specs = [spec for spec in concrete_specs.values() if "external" not in spec]' +assert_contains "$container_validator" 'if source_targets != {"x86_64_v4"}:' +assert_contains "$container_validator" 'source-built Spack specs do not all target x86_64_v4' +if grep -Fq -- 'Spack concretization does not contain target=zen4' "$container_validator"; then + fail 'validator still requires the non-concrete target spelling zen4' +fi + +# h5pcc links shared HDF5 without embedding its installation path. The runtime +# check must add only the verified HDF5 prefix and retain strict MPI resolution. +assert_contains "$container_validator" '[[ "$h5pcc" == "$tpl_root"/* ]] || die "h5pcc resolves outside the TPL root: $h5pcc"' +assert_contains "$container_validator" 'compgen -G "$candidate/libhdf5.so*" >/dev/null || continue' +assert_contains "$container_validator" 'export LD_LIBRARY_PATH="$hdf5_loader_path${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"' +assert_contains "$container_validator" 'die "$target resolves libmpi outside $mpi_prefix: $library"' + +# Reproduce Open MPI's symlink-sensitive wrapper layout. Invoking opal_wrapper +# directly fails; invoking the mpicc symlink succeeds because argv[0] is mpicc. +wrapper_root="${test_root}/wrapper" +mkdir -p "$wrapper_root" +cat >"${wrapper_root}/opal_wrapper" <<'EOF' +#!/usr/bin/env bash +if [[ ${0##*/} == mpicc && ${1:-} == --showme ]]; then + printf 'gcc -march=native -mtune=native -lmpi\n' + exit 0 +fi +printf 'wrapper invoked with the wrong identity or arguments\n' >&2 +exit 23 +EOF +chmod +x "${wrapper_root}/opal_wrapper" +ln -s opal_wrapper "${wrapper_root}/mpicc" + +if "${wrapper_root}/opal_wrapper" --showme >/dev/null 2>&1; then + fail 'symlink-sensitive fixture unexpectedly accepted direct opal_wrapper invocation' +fi +bash "$capture_wrapper" openmpi "${wrapper_root}/mpicc" "${test_root}/openmpi-show.txt" +assert_contains "${test_root}/openmpi-show.txt" '-march=native -mtune=native' + +# Model Spack with a deterministic command log: the first two source fetches +# fail, the third succeeds, and only then may the no-buildcache install run. +fake_root="${test_root}/fake-spack" +mkdir -p "${fake_root}/environment" +printf 'spack: {}\n' >"${fake_root}/environment/spack.yaml" +cat >"${fake_root}/spack" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +: "${FAKE_SPACK_LOG:?}" +: "${FAKE_SPACK_COUNT:?}" +printf '%s\n' "$*" >>"$FAKE_SPACK_LOG" +case " $* " in + *' fetch '*) + count=0 + [[ ! -f "$FAKE_SPACK_COUNT" ]] || count=$(<"$FAKE_SPACK_COUNT") + count=$((count + 1)) + printf '%d\n' "$count" >"$FAKE_SPACK_COUNT" + (( count >= ${FAKE_SPACK_SUCCEED_ON:-1} )) + ;; +esac +EOF +chmod +x "${fake_root}/spack" + +export FAKE_SPACK_LOG="${fake_root}/commands.log" +export FAKE_SPACK_COUNT="${fake_root}/fetch-count" +export FAKE_SPACK_SUCCEED_ON=3 +HBV4_FETCH_ATTEMPTS=3 HBV4_FETCH_RETRY_DELAY_SECONDS=0 \ + bash "$install_from_source" \ + "${fake_root}/spack" "${fake_root}/environment" lvarray_hostconfig + +[[ $(<"$FAKE_SPACK_COUNT") == 3 ]] || fail 'source prefetch did not retry exactly three times' +assert_contains "$FAKE_SPACK_LOG" '-D' +assert_contains "$FAKE_SPACK_LOG" 'concretize --fresh' +assert_contains "$FAKE_SPACK_LOG" '-k fetch --dependencies' +assert_contains "$FAKE_SPACK_LOG" '-k install --fresh --keep-stage --no-cache -p 4 -j 176 -u lvarray_hostconfig' + +# Exhausting the bounded retry budget must fail closed and never compile. +: >"$FAKE_SPACK_LOG" +rm -f "$FAKE_SPACK_COUNT" +export FAKE_SPACK_SUCCEED_ON=99 +if HBV4_FETCH_ATTEMPTS=2 HBV4_FETCH_RETRY_DELAY_SECONDS=0 \ + bash "$install_from_source" \ + "${fake_root}/spack" "${fake_root}/environment" lvarray_hostconfig; then + fail 'source prefetch unexpectedly succeeded after exhausting its retry budget' +fi +if grep -Fq -- ' install ' "$FAKE_SPACK_LOG"; then + fail 'Spack install ran after source prefetch exhausted its retry budget' +fi + +printf 'HBv4 build resilience tests passed.\n' diff --git a/ci/azure/tests/test-hbv4-local-nvme.sh b/ci/azure/tests/test-hbv4-local-nvme.sh new file mode 100755 index 00000000..c77135aa --- /dev/null +++ b/ci/azure/tests/test-hbv4-local-nvme.sh @@ -0,0 +1,454 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2016 +# +# Mocked HBv4 local-NVMe safety fixtures +# -------------------------------------- +# Each fixture supplies regular files plus PATH-injected command shims to the +# production helper. Discovery deliberately models the Azure live shape: +# lsblk enumerates namespace paths while `nvme id-ctrl` returns the padded +# multiword controller model independently. Negative cases model one malformed +# or ambiguous storage state at a time and assert that the validation boundary +# prevents mdadm create, mkfs, and mount. No fixture can address a host block +# device. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +HELPER="${REPO_ROOT}/ci/azure/scripts/setup-hbv4-local-nvme.sh" +TEST_ROOT="$(mktemp -d)" +trap 'rm -rf "${TEST_ROOT}"' EXIT + +EXPECTED_BYTES=$(( 1831054 * 1024 * 1024 )) +failures=0 + +fail() +{ + echo "FAIL: $*" >&2 + failures=$(( failures + 1 )) +} + +make_mock() +{ + local case_dir="$1" + local command_name="$2" + shift 2 + { + echo '#!/usr/bin/env bash' + echo 'set -euo pipefail' + printf '%s\n' "$@" + } > "${case_dir}/bin/${command_name}" + chmod +x "${case_dir}/bin/${command_name}" +} + +new_case() +{ + local name="$1" + CASE_DIR="${TEST_ROOT}/${name}" + mkdir -p \ + "${CASE_DIR}/bin" \ + "${CASE_DIR}/dev/disk/by-id" \ + "${CASE_DIR}/dev/disk/azure/local/by-serial" \ + "${CASE_DIR}/dev/md" \ + "${CASE_DIR}/sys/class/block" \ + "${CASE_DIR}/mount" \ + "${CASE_DIR}/run" + : > "${CASE_DIR}/mutations.log" + printf 'Filename Type Size Used Priority\n' > "${CASE_DIR}/swaps" + printf '%s\n' "/dev/osdisk" > "${CASE_DIR}/protected-root" + printf '%s\n' "/dev/osdisk" > "${CASE_DIR}/chain-osdisk" + printf '%s\n' "/dev/nvme0n1" "/dev/nvme1n1" \ + > "${CASE_DIR}/discovery" + for device in nvme0n1 nvme1n1; do + : > "${CASE_DIR}/dev/${device}" + mkdir -p "${CASE_DIR}/sys/class/block/${device}/holders" + printf 'disk\n' > "${CASE_DIR}/type-${device}" + printf '%s\n' "${EXPECTED_BYTES}" > "${CASE_DIR}/size-${device}" + printf 'Microsoft NVMe Direct Disk\n' > "${CASE_DIR}/model-${device}" + printf '%s disk\n' "${CASE_DIR}/dev/${device}" > "${CASE_DIR}/topology-${device}" + ln -s "../../${device}" "${CASE_DIR}/dev/disk/by-id/azure-${device}" + ln -s "../../../../${device}" \ + "${CASE_DIR}/dev/disk/azure/local/by-serial/serial-${device}" + done + + make_mock "${CASE_DIR}" id \ + 'if [[ "${1:-}" == "-u" ]]; then echo "${MOCK_UID:-0}"; else /usr/bin/id "$@"; fi' + make_mock "${CASE_DIR}" flock \ + '[[ "${MOCK_FORCE_LOCKED:-0}" != 1 ]]' + make_mock "${CASE_DIR}" lsblk \ + 'args=" $* "' \ + 'if [[ "${args}" == *" -dnpo NAME "* ]]; then cat "${MOCK_CASE_DIR}/discovery"; exit 0; fi' \ + 'if [[ "${args}" == *" -srnpo "* ]]; then' \ + ' source_device="${!#}"; chain_file="${MOCK_CASE_DIR}/chain-$(basename "${source_device}")"' \ + ' [[ -f "${chain_file}" ]] || exit 1' \ + ' cat "${chain_file}"; exit 0' \ + 'fi' \ + 'device="${!#}"; device_base="$(basename "${device}")"' \ + 'if [[ "${args}" == *" -dno TYPE "* ]]; then cat "${MOCK_CASE_DIR}/type-${device_base}"; exit 0; fi' \ + 'if [[ "${args}" == *" -dnbo SIZE "* ]]; then cat "${MOCK_CASE_DIR}/size-${device_base}"; exit 0; fi' \ + 'device="${!#}"; cat "${MOCK_CASE_DIR}/topology-$(basename "${device}")"' + make_mock "${CASE_DIR}" nvme \ + '[[ "${1:-}" == "id-ctrl" ]] || exit 99' \ + 'device_base="$(basename "${2}")"' \ + '[[ ",${MOCK_NVME_ID_FAILURES:-}," != *",${device_base},"* ]] || exit 1' \ + 'printf "vid : 0x1414\nmn : %s \nfr : 1.0\n" "$(<"${MOCK_CASE_DIR}/model-${device_base}")"' + make_mock "${CASE_DIR}" findmnt \ + 'args=" $* "' \ + 'mount_arg="${!#}"' \ + 'if [[ "${args}" == *" SOURCE,FSTYPE,OPTIONS "* ]]; then' \ + ' [[ "${MOCK_MOUNTED:-0}" == 1 ]] || exit 1' \ + ' printf "%s xfs %s\n" "${MOCK_MD_DEVICE}" "${MOCK_MOUNT_OPTIONS:-rw,noatime}"' \ + ' exit 0' \ + 'fi' \ + 'if [[ "${mount_arg}" == "/" ]]; then cat "${MOCK_CASE_DIR}/protected-root"; exit 0; fi' \ + 'if [[ -f "${MOCK_CASE_DIR}/protected-$(basename "${mount_arg}")" ]]; then' \ + ' cat "${MOCK_CASE_DIR}/protected-$(basename "${mount_arg}")"; exit 0' \ + 'fi' \ + 'exit 1' + make_mock "${CASE_DIR}" blkid \ + 'if [[ "${1:-}" == "-p" ]]; then' \ + ' [[ ",${MOCK_SIGNATURE_DEVICES:-}," == *",$(basename "${2}"),"* ]] && exit 0' \ + ' exit 2' \ + 'fi' \ + 'if [[ "$*" == *"-s TYPE"* ]]; then printf "%s\n" "${MOCK_MD_TYPE:-xfs}"; exit 0; fi' \ + 'if [[ "$*" == *"-s LABEL"* ]]; then printf "%s\n" "${MOCK_MD_LABEL:-hbv4-local}"; exit 0; fi' \ + 'exit 2' + make_mock "${CASE_DIR}" mdadm \ + 'case "${1:-}" in' \ + ' --examine)' \ + ' [[ ",${MOCK_RAID_METADATA_DEVICES:-}," == *",$(basename "${2}"),"* ]] && exit 0' \ + ' exit 1' \ + ' ;;' \ + ' --detail)' \ + ' printf "MD_LEVEL=%s\nMD_DEVICES=%s\n" "${MOCK_MD_LEVEL:-raid0}" "${MOCK_MD_DEVICES:-2}"' \ + ' ;;' \ + ' --create)' \ + ' printf "mdadm %s\n" "$*" >> "${MOCK_CASE_DIR}/mutations.log"' \ + ' ;;' \ + ' *) exit 99 ;;' \ + 'esac' + make_mock "${CASE_DIR}" mkfs.xfs \ + 'printf "mkfs.xfs %s\n" "$*" >> "${MOCK_CASE_DIR}/mutations.log"' + make_mock "${CASE_DIR}" mount \ + 'printf "mount %s\n" "$*" >> "${MOCK_CASE_DIR}/mutations.log"' + make_mock "${CASE_DIR}" mountpoint \ + '[[ "${MOCK_MOUNTED:-0}" == 1 ]]' + make_mock "${CASE_DIR}" xfs_info \ + 'printf "naming =version 2 bsize=4096 ascii-ci=0, ftype=%s\n" "${MOCK_XFS_FTYPE:-1}"' +} + +run_helper() +{ + local case_dir="$1" + shift + local -a helper_args=() + while (( $# > 0 )) && [[ "$1" == --* ]]; do + helper_args+=("$1") + shift + done + env \ + PATH="${case_dir}/bin:${PATH}" \ + MOCK_CASE_DIR="${case_dir}" \ + MOCK_MD_DEVICE="${case_dir}/dev/md/hbv4-local" \ + GEOS_HBV4_TEST_MODE=1 \ + GEOS_HBV4_TEST_DEV_ROOT="${case_dir}/dev" \ + GEOS_HBV4_TEST_BY_ID_DIR="${case_dir}/dev/disk/by-id" \ + GEOS_HBV4_TEST_AZURE_DISK_DIR="${case_dir}/dev/disk/azure" \ + GEOS_HBV4_TEST_AZURE_LOCAL_BY_SERIAL_DIR="${case_dir}/dev/disk/azure/local/by-serial" \ + GEOS_HBV4_TEST_SYS_CLASS_BLOCK_DIR="${case_dir}/sys/class/block" \ + GEOS_HBV4_TEST_PROC_SWAPS="${case_dir}/swaps" \ + GEOS_HBV4_TEST_MD_DEVICE="${case_dir}/dev/md/hbv4-local" \ + GEOS_HBV4_TEST_MOUNT_POINT="${case_dir}/mount/hbv4-local" \ + GEOS_HBV4_TEST_LOCK_FILE="${case_dir}/run/setup.lock" \ + "$@" \ + bash "${HELPER}" "${helper_args[@]}" +} + +assert_no_destructive_mutation() +{ + local case_dir="$1" + if grep -Eq '^(mdadm|mkfs\.xfs|mount) ' "${case_dir}/mutations.log"; then + fail "$(basename "${case_dir}") executed a destructive command after rejection" + fi +} + +expect_success() +{ + local name="$1" + shift + new_case "${name}" + if ! run_helper "${CASE_DIR}" "$@" >"${CASE_DIR}/stdout" 2>"${CASE_DIR}/stderr"; then + fail "${name} unexpectedly failed: $(<"${CASE_DIR}/stderr")" + fi +} + +expect_rejection() +{ + local name="$1" + shift + new_case "${name}" + if run_helper "${CASE_DIR}" "$@" >"${CASE_DIR}/stdout" 2>"${CASE_DIR}/stderr"; then + fail "${name} unexpectedly succeeded" + fi + assert_no_destructive_mutation "${CASE_DIR}" +} + +# A pristine pair succeeds, uses stable links in lexical order, and performs the +# one permitted create/format/mount sequence. Its accepted-device diagnostics +# are also a regression for the live Azure shape: namespace-only lsblk output +# plus a padded `Microsoft NVMe Direct Disk` Identify Controller field. +expect_success fresh_success +if [[ "$(grep -c 'decision=accepted' "${CASE_DIR}/stdout")" != "2" ]]; then + fail "fresh success did not report both accepted live-shape NVMe namespaces" +fi +grep -Fq 'model=Microsoft\ NVMe\ Direct\ Disk' "${CASE_DIR}/stdout" || + fail "fresh success did not preserve the multiword controller model" +grep -Fq "size_bytes=${EXPECTED_BYTES}" "${CASE_DIR}/stdout" || + fail "fresh success did not report the observed byte size" +if ! grep -Fq \ + "${CASE_DIR}/dev/disk/azure/local/by-serial/serial-nvme0n1 ${CASE_DIR}/dev/disk/azure/local/by-serial/serial-nvme1n1" \ + "${CASE_DIR}/mutations.log"; then + fail "fresh success did not prefer deterministic Azure serial member links" +fi +for operation in mdadm mkfs.xfs mount; do + grep -q "^${operation} " "${CASE_DIR}/mutations.log" || + fail "fresh success omitted ${operation}" +done +grep -Fq "LABEL=hbv4-local ${CASE_DIR}/mount/hbv4-local" "${CASE_DIR}/mutations.log" || + fail "fresh success did not mount the XFS filesystem by label" + +# Generic by-id aliases remain a deterministic fallback for images where the +# Azure local/by-serial directory has not been populated. +new_case by_id_fallback +rm "${CASE_DIR}/dev/disk/azure/local/by-serial/"* +if ! run_helper "${CASE_DIR}" >"${CASE_DIR}/stdout" 2>"${CASE_DIR}/stderr"; then + fail "by-id fallback unexpectedly failed: $(<"${CASE_DIR}/stderr")" +fi +grep -Fq \ + "${CASE_DIR}/dev/disk/by-id/azure-nvme0n1 ${CASE_DIR}/dev/disk/by-id/azure-nvme1n1" \ + "${CASE_DIR}/mutations.log" || + fail "by-id fallback was not deterministic" + +# Dry-run reports all planned state changes but no command shim sees a mutation. +expect_success dry_run --dry-run +assert_no_destructive_mutation "${CASE_DIR}" +grep -q 'DRY-RUN: mdadm' "${CASE_DIR}/stdout" || + fail "dry run did not report mdadm creation" +grep -q 'DRY-RUN: mkfs.xfs' "${CASE_DIR}/stdout" || + fail "dry run did not report XFS formatting" +grep -q 'DRY-RUN: mount' "${CASE_DIR}/stdout" || + fail "dry run did not report mounting" + +# Candidate cardinality is an identity invariant: too few or too many matching +# devices is ambiguous and must stop before inspecting or mutating members. +for count in 0 1 3; do + new_case "candidate_count_${count}" + : > "${CASE_DIR}/discovery" + for (( index=0; index "${CASE_DIR}/dev/${device}" + mkdir -p "${CASE_DIR}/sys/class/block/${device}/holders" + printf 'disk\n' > "${CASE_DIR}/type-${device}" + printf '%s\n' "${EXPECTED_BYTES}" > "${CASE_DIR}/size-${device}" + printf 'Microsoft NVMe Direct Disk\n' > "${CASE_DIR}/model-${device}" + printf '%s disk\n' "${CASE_DIR}/dev/${device}" > "${CASE_DIR}/topology-${device}" + printf '/dev/%s\n' "${device}" >> "${CASE_DIR}/discovery" + done + if run_helper "${CASE_DIR}" >"${CASE_DIR}/stdout-2" 2>"${CASE_DIR}/stderr-2"; then + fail "candidate_count_${count} unexpectedly succeeded" + fi + assert_no_destructive_mutation "${CASE_DIR}" +done + +# Identity mismatches remain ordinary rejected observations until the exact-two +# guard fails. These fixtures protect both the model/size allowlist and the +# actionable diagnostics needed to distinguish a platform-shape change from a +# topology or signature rejection. +new_case rejected_controller_model +printf 'Unexpected NVMe Device\n' > "${CASE_DIR}/model-nvme1n1" +if run_helper "${CASE_DIR}" >"${CASE_DIR}/stdout" 2>"${CASE_DIR}/stderr"; then + fail "unexpected controller model succeeded" +fi +assert_no_destructive_mutation "${CASE_DIR}" +grep -Fq "device=${CASE_DIR}/dev/nvme1n1" "${CASE_DIR}/stderr" || + fail "controller-model diagnostic omitted the guarded device root" +grep -Fq 'model=Unexpected\ NVMe\ Device' "${CASE_DIR}/stderr" || + fail "controller-model rejection omitted the observed model" +grep -Fq 'reason=controller\ model\ does\ not\ match' "${CASE_DIR}/stderr" || + fail "controller-model rejection omitted its reason" + +new_case rejected_byte_size +printf '%s\n' "$(( EXPECTED_BYTES + EXPECTED_BYTES / 100 + 1 ))" \ + > "${CASE_DIR}/size-nvme1n1" +if run_helper "${CASE_DIR}" >"${CASE_DIR}/stdout" 2>"${CASE_DIR}/stderr"; then + fail "out-of-tolerance byte size succeeded" +fi +assert_no_destructive_mutation "${CASE_DIR}" +grep -Fq 'reason=byte\ size\ is\ outside' "${CASE_DIR}/stderr" || + fail "byte-size rejection omitted its reason" + +# Each negative fixture below models a state that could indicate user or OS +# ownership. The correct behavior is refusal, never automatic cleanup. +new_case os_overlap +printf '%s\n' "${CASE_DIR}/dev/nvme0n1" > "${CASE_DIR}/chain-osdisk" +if run_helper "${CASE_DIR}" >/dev/null 2>&1; then fail "OS overlap succeeded"; fi +assert_no_destructive_mutation "${CASE_DIR}" + +new_case resource_overlap +printf '%s\n' "${CASE_DIR}/dev/resource" > "${CASE_DIR}/protected-mnt" +printf '%s\n' "${CASE_DIR}/dev/nvme1n1" > "${CASE_DIR}/chain-resource" +if run_helper "${CASE_DIR}" >/dev/null 2>&1; then fail "resource overlap succeeded"; fi +assert_no_destructive_mutation "${CASE_DIR}" + +new_case unmounted_azure_os_overlap +ln -s "../../nvme0n1" "${CASE_DIR}/dev/disk/azure/os" +printf '%s\n' "${CASE_DIR}/dev/nvme0n1" > "${CASE_DIR}/chain-os" +if run_helper "${CASE_DIR}" >/dev/null 2>&1; then + fail "unmounted explicit Azure OS disk overlap succeeded" +fi +assert_no_destructive_mutation "${CASE_DIR}" + +new_case unmounted_azure_resource_overlap +ln -s "../../nvme1n1" "${CASE_DIR}/dev/disk/azure/resource" +printf '%s\n' "${CASE_DIR}/dev/nvme1n1" > "${CASE_DIR}/chain-resource" +if run_helper "${CASE_DIR}" >/dev/null 2>&1; then + fail "unmounted explicit Azure resource disk overlap succeeded" +fi +assert_no_destructive_mutation "${CASE_DIR}" + +new_case mounted_candidate +printf '%s disk /scratch\n' "${CASE_DIR}/dev/nvme0n1" > "${CASE_DIR}/topology-nvme0n1" +if run_helper "${CASE_DIR}" >/dev/null 2>&1; then fail "mounted candidate succeeded"; fi +assert_no_destructive_mutation "${CASE_DIR}" + +new_case partitioned_candidate +printf '%s disk\n%s partition\n' \ + "${CASE_DIR}/dev/nvme0n1" "${CASE_DIR}/dev/nvme0n1p1" \ + > "${CASE_DIR}/topology-nvme0n1" +if run_helper "${CASE_DIR}" >/dev/null 2>&1; then fail "partitioned candidate succeeded"; fi +assert_no_destructive_mutation "${CASE_DIR}" + +new_case swap_candidate +printf '%s partition 1 1 -2\n' "${CASE_DIR}/dev/nvme0n1" >> "${CASE_DIR}/swaps" +if run_helper "${CASE_DIR}" >/dev/null 2>&1; then fail "swap candidate succeeded"; fi +assert_no_destructive_mutation "${CASE_DIR}" + +new_case holder_candidate +: > "${CASE_DIR}/sys/class/block/nvme0n1/holders/dm-0" +if run_helper "${CASE_DIR}" >/dev/null 2>&1; then fail "held candidate succeeded"; fi +assert_no_destructive_mutation "${CASE_DIR}" + +expect_rejection filesystem_signature MOCK_SIGNATURE_DEVICES=nvme0n1 +expect_rejection foreign_raid_metadata MOCK_RAID_METADATA_DEVICES=nvme1n1 + +# Destination ambiguity must also be rejected before mdadm. An unrelated mount, +# a nonempty directory exposed by a lost mount, or a symlink could otherwise +# hide or redirect OS-disk state after the candidate disks had been formatted. +expect_rejection unrelated_destination_mount MOCK_MOUNTED=1 + +new_case nonempty_destination +mkdir -p "${CASE_DIR}/mount/hbv4-local" +: > "${CASE_DIR}/mount/hbv4-local/unowned-state" +if run_helper "${CASE_DIR}" >/dev/null 2>&1; then + fail "nonempty unmounted destination succeeded" +fi +assert_no_destructive_mutation "${CASE_DIR}" + +new_case symlink_destination +mkdir -p "${CASE_DIR}/mount/other" +ln -s "${CASE_DIR}/mount/other" "${CASE_DIR}/mount/hbv4-local" +if run_helper "${CASE_DIR}" >/dev/null 2>&1; then + fail "symbolic-link destination succeeded" +fi +assert_no_destructive_mutation "${CASE_DIR}" + +prepare_existing_array() +{ + local case_dir="$1" + : > "${case_dir}/dev/md/hbv4-local" + mkdir -p \ + "${case_dir}/sys/class/block/hbv4-local/md" \ + "${case_dir}/sys/class/block/hbv4-local/slaves" + printf 'raid0\n' > "${case_dir}/sys/class/block/hbv4-local/md/level" + printf '2\n' > "${case_dir}/sys/class/block/hbv4-local/md/raid_disks" + # Model the real RAID-0 sysfs contract: geometry and active slaves exist, but + # the redundancy-only md/degraded attribute does not. + : > "${case_dir}/sys/class/block/hbv4-local/slaves/nvme0n1" + : > "${case_dir}/sys/class/block/hbv4-local/slaves/nvme1n1" + for device in nvme0n1 nvme1n1; do + printf '%s disk\n%s raid0\n' \ + "${case_dir}/dev/${device}" "${case_dir}/dev/md/hbv4-local" \ + > "${case_dir}/topology-${device}" + : > "${case_dir}/sys/class/block/${device}/holders/hbv4-local" + done +} + +# A complete array is idempotent. An unmounted instance is mounted without +# recreation; an already-correct mount performs no destructive operation. +new_case existing_unmounted +prepare_existing_array "${CASE_DIR}" +if ! run_helper "${CASE_DIR}" >"${CASE_DIR}/stdout" 2>"${CASE_DIR}/stderr"; then + fail "complete unmounted array was rejected: $(<"${CASE_DIR}/stderr")" +fi +grep -q '^mount ' "${CASE_DIR}/mutations.log" || + fail "complete unmounted array was not mounted" +if grep -Eq '^(mdadm|mkfs\.xfs) ' "${CASE_DIR}/mutations.log"; then + fail "complete unmounted array was recreated" +fi + +new_case existing_mounted +prepare_existing_array "${CASE_DIR}" +if ! run_helper "${CASE_DIR}" MOCK_MOUNTED=1 >"${CASE_DIR}/stdout" 2>"${CASE_DIR}/stderr"; then + fail "complete mounted array was rejected: $(<"${CASE_DIR}/stderr")" +fi +assert_no_destructive_mutation "${CASE_DIR}" + +# Existing arrays must fail closed when geometry, active membership, exact +# members, filesystem, label, source, or mount options differ from the contract. +for fixture in partial_array foreign_members stale_label wrong_filesystem wrong_ftype wrong_mount_options; do + new_case "${fixture}" + prepare_existing_array "${CASE_DIR}" + case "${fixture}" in + partial_array) + # Keep the declared two-device geometry while omitting one active slave; + # the helper must reject this inconsistent member topology. + rm "${CASE_DIR}/sys/class/block/hbv4-local/slaves/nvme1n1" + fixture_env=() + ;; + foreign_members) + rm "${CASE_DIR}/sys/class/block/hbv4-local/slaves/nvme1n1" + : > "${CASE_DIR}/sys/class/block/hbv4-local/slaves/nvme9n1" + fixture_env=() + ;; + stale_label) + fixture_env=(MOCK_MD_LABEL=other) + ;; + wrong_filesystem) + fixture_env=(MOCK_MD_TYPE=ext4) + ;; + wrong_ftype) + fixture_env=(MOCK_XFS_FTYPE=0) + ;; + wrong_mount_options) + fixture_env=("MOCK_MOUNTED=1" "MOCK_MOUNT_OPTIONS=rw,relatime") + ;; + esac + if run_helper "${CASE_DIR}" "${fixture_env[@]}" >/dev/null 2>&1; then + fail "${fixture} unexpectedly succeeded" + fi + assert_no_destructive_mutation "${CASE_DIR}" +done + +# The flock shim returns the same immediate failure as `flock -n` under real +# contention. This keeps the fixture portable to macOS, where util-linux flock +# is absent, without weakening the production nonblocking invocation contract. +expect_rejection lock_contention MOCK_FORCE_LOCKED=1 + +expect_rejection non_root MOCK_UID=1000 + +if (( failures > 0 )); then + echo "${failures} HBv4 local-NVMe fixture(s) failed." >&2 + exit 1 +fi + +echo "HBv4 local-NVMe fixtures passed." diff --git a/ci/azure/tests/test-hbv4-workflow-contract.sh b/ci/azure/tests/test-hbv4-workflow-contract.sh new file mode 100755 index 00000000..c00049cf --- /dev/null +++ b/ci/azure/tests/test-hbv4-workflow-contract.sh @@ -0,0 +1,370 @@ +#!/usr/bin/env bash +# Validate the HBv4 workflow trust boundary without contacting GitHub or Azure. + +set -euo pipefail + +repository_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../.." && pwd -P)" + +ruby - "${repository_root}" <<'RUBY' +require "json" +require "yaml" + +root = ARGV.fetch(0) +workflow_path = File.join(root, ".github/workflows/_docker_build_tpls_hbv4_provider.yml") +entry_workflow_path = File.join(root, ".github/workflows/docker_build_tpls_hbv4.yml") +janitor_workflow_path = File.join(root, ".github/workflows/azure-janitor.yml") +provision_path = File.join(root, "ci/azure/actions/provision-hbv4-runner/action.yml") +cleanup_path = File.join(root, "ci/azure/actions/cleanup-runner/action.yml") +bootstrap_path = File.join(root, "ci/azure/scripts/runner-bootstrap.sh") +host_validator_path = File.join(root, "scripts/hbv4/validate-host.sh") +provider_manifest_paths = %w[openmpi mpich].to_h do |provider| + [provider, File.join(root, "docker/ubuntu-hbv4-#{provider}-spack.yaml")] +end + +def assert!(condition, message) + raise "HBv4 workflow contract failure: #{message}" unless condition +end + +def load_yaml(path) + assert!(File.file?(path), "missing #{path}") + value = YAML.safe_load(File.read(path), aliases: true, filename: path) + assert!(value.is_a?(Hash), "#{path} must contain a YAML mapping") + value +end + +def steps_for(job) + steps = job.fetch("steps", []) + assert!(steps.is_a?(Array), "job steps must be an array") + steps +end + +def action_steps(action) + steps = action.fetch("runs", {}).fetch("steps", []) + assert!(steps.is_a?(Array), "composite action steps must be an array") + steps +end + +def find_one_step(steps, description) + matches = steps.select { |step| yield(step) } + assert!(matches.length == 1, "expected exactly one #{description}, found #{matches.length}") + matches.first +end + +def shell_without_comments(script) + script.each_line.reject { |line| line.strip.start_with?("#") }.join +end + +def find_steps_using(value, action_prefix, matches = []) + case value + when Hash + uses = value["uses"] + matches << value if uses.is_a?(String) && uses.start_with?(action_prefix) + value.each_value { |child| find_steps_using(child, action_prefix, matches) } + when Array + value.each { |child| find_steps_using(child, action_prefix, matches) } + end + matches +end + +workflow = load_yaml(workflow_path) +entry_workflow = load_yaml(entry_workflow_path) +janitor_workflow = load_yaml(janitor_workflow_path) +provision = load_yaml(provision_path) +cleanup = load_yaml(cleanup_path) +jobs = workflow.fetch("jobs") + +# Keep JavaScript actions on their reviewed Node 24 revisions and prevent the +# deprecated numeric GitHub App ID input from returning anywhere in the Azure +# control plane. +auth_documents = [workflow, janitor_workflow, provision, cleanup] +azure_login_ref = "azure/login@f5d393ae46f8fde4be8b75f32e3fc50e654ad0ca" +azure_login_steps = auth_documents.flat_map { |document| find_steps_using(document, "azure/login@") } +assert!(azure_login_steps.length == 5, + "expected all five Azure login call sites, found #{azure_login_steps.length}") +assert!(azure_login_steps.all? { |step| step["uses"] == azure_login_ref }, + "every Azure login must use the reviewed Node 24 v3.0.1 commit") + +app_token_ref = + "actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1" +app_token_steps = auth_documents.flat_map do |document| + find_steps_using(document, "actions/create-github-app-token@") +end +assert!(app_token_steps.length == 3, + "expected all three GitHub App token call sites, found #{app_token_steps.length}") +app_token_steps.each do |step| + inputs = step.fetch("with", {}) + assert!(step["uses"] == app_token_ref, + "GitHub App token action must remain immutable and Node 24 compatible") + assert!(inputs.key?("client-id") && !inputs.key?("app-id"), + "GitHub App token action must use client-id, never deprecated app-id") +end + +auth_contract = JSON.generate(auth_documents) +assert!(!auth_contract.include?("AZURE_RUNNER_APP_ID") && + !auth_contract.include?("runner_app_id") && + !auth_contract.include?('"app-id"'), + "legacy numeric GitHub App ID wiring must be absent") +assert!(auth_contract.include?("AZURE_RUNNER_APP_CLIENT_ID") && + auth_contract.include?("runner_app_client_id"), + "GitHub App client ID variable must reach both composite actions") + +# The trusted entry workflow must run the checked-in Azure fixtures before it +# authorizes either provider lifecycle to cross the paid provisioning boundary. +authorize_steps = steps_for(entry_workflow.fetch("jobs").fetch("authorize")) +control_checkout_index = authorize_steps.index do |step| + step["name"] == "Checkout trusted HBv4 control-plane tests" +end +control_test_index = authorize_steps.index do |step| + step["name"] == "Validate trusted HBv4 control plane" +end +source_index = authorize_steps.index do |step| + step["name"] == "Validate trigger and live source identity" +end +assert!(!control_checkout_index.nil? && !control_test_index.nil? && !source_index.nil?, + "entry workflow must checkout, test, and authorize its trusted control plane") +assert!(control_checkout_index < control_test_index && control_test_index < source_index, + "trusted Azure fixtures must run before source authorization") +control_checkout = authorize_steps.fetch(control_checkout_index) +control_sparse_checkout = control_checkout.fetch("with", {}).fetch("sparse-checkout", "") +control_sparse_paths = control_sparse_checkout.lines.map(&:strip) +assert!(control_sparse_paths.include?("docker/tpl-ubuntu.Dockerfile"), + "trusted control-plane checkout must include the Dockerfile inspected by resilience tests") +%w[openmpi mpich].each do |provider| + manifest = "docker/ubuntu-hbv4-#{provider}-spack.yaml" + assert!(control_sparse_paths.include?(manifest), + "trusted control-plane checkout must include #{manifest} inspected by contract tests") +end +control_test_script = shell_without_comments(authorize_steps.fetch(control_test_index).fetch("run")) +assert!(control_test_script.include?("ci/azure/tests/test-*.sh") && + control_test_script.include?("test-render-runner-bootstrap.py"), + "entry workflow must run all shell and Python Azure fixtures") + +{ + "provision_builder" => "builder", + "provision_tester" => "tester", +}.each do |job_name, role| + job = jobs.fetch(job_name) + permissions = job.fetch("permissions", {}) + assert!(permissions["contents"] == "read", "#{job_name} must grant contents: read") + assert!(permissions["pull-requests"] == "read", + "#{job_name} must grant pull-requests: read for live PR authorization") + assert!(permissions["id-token"] == "write", "#{job_name} must retain Azure OIDC") + + steps = steps_for(job) + provision_index = steps.index do |step| + step["uses"] == "./ci/azure/actions/provision-hbv4-runner" + end + assert!(!provision_index.nil?, "#{job_name} must call the local provision action") + assert!(provision_index.positive?, "#{job_name} must authorize before provisioning") + authorization = steps.fetch(provision_index - 1) + assert!(authorization.fetch("name", "").include?("Reauthorize live source"), + "#{job_name} source authorization must immediately precede provisioning") + + authorization_env = authorization.fetch("env", {}) + assert!(authorization_env["GH_TOKEN"] == "${{ github.token }}", + "#{job_name} source authorization must use the read-only job token") + assert!(!authorization_env.values.include?("${{ steps.runner_app_token.outputs.token }}"), + "#{job_name} must not expose the runner-administration App token") + %w[EVENT_NAME EVENT_REF PR_NUMBER SOURCE_SHA].each do |key| + assert!(authorization_env.key?(key), "#{job_name} source authorization omitted #{key}") + end + + script = shell_without_comments(authorization.fetch("run")) + assert!(script.include?("pull_request_target") && script.include?("workflow_dispatch"), + "#{job_name} must preserve PR-label and manual-dispatch routes") + assert!(script.include?('/pulls/${PR_NUMBER}') && script.include?('/commits/${SOURCE_SHA}'), + "#{job_name} must read live PR and exact commit state") + assert!(script.include?('.state == "open"') && + script.include?(".head.repo.full_name == $repository") && + script.include?(".head.sha == $sha") && + script.include?("ci: build HBv4 TPL"), + "#{job_name} PR route must require open, same-repository, exact-head, labelled state") + assert!(script.include?('refs/pull/${PR_NUMBER}/head'), + "#{job_name} must bind PR authorization to the canonical head ref") + assert!(script.include?('.sha // empty') && script.include?('== "${SOURCE_SHA}"'), + "#{job_name} dispatch route must require the exact repository commit") + + provision_inputs = steps.fetch(provision_index).fetch("with", {}) + assert!(provision_inputs["runner_role"] == role, + "#{job_name} must pass its exact lifecycle role") +end + +provision_steps = action_steps(provision) +provision_text = provision_steps.map { |step| shell_without_comments(step.fetch("run", "")) }.join("\n") +assert!(!provision_text.include?("/pulls/") && !provision_text.include?("/commits/"), + "provision composite must not perform source authorization with the App token") +assert!(!JSON.generate(provision).include?("github.token"), + "provision composite must not receive the workflow source-read token") + +provision_steps.each do |step| + environment = step.fetch("env", {}) + assert!(!environment.key?("RUNNER_NAME"), + "provision step '#{step.fetch("name", "unnamed")}' overrides protected RUNNER_NAME") + script = shell_without_comments(step.fetch("run", "")) + assert!(!script.include?('${RUNNER_NAME}'), + "provision step '#{step.fetch("name", "unnamed")}' reads protected RUNNER_NAME") + + next unless environment.key?("GH_TOKEN") + + assert!(environment["GH_TOKEN"] == "${{ steps.runner_app_token.outputs.token }}", + "provision composite GH_TOKEN must always be the runner-administration App token") + endpoints = script.scan(%r{repos/\$\{GITHUB_REPOSITORY\}/[^"'[:space:]]+}) + assert!(!endpoints.empty?, "App-token step must invoke an explicit repository runner endpoint") + assert!(endpoints.all? { |endpoint| endpoint.start_with?("repos/${GITHUB_REPOSITORY}/actions/runners") }, + "App token escaped the repository runner API: #{endpoints.join(', ')}") +end + +create_vm = find_one_step(provision_steps, "paid VM creation step") do |step| + step["id"] == "create_vm" +end +create_vm_env = create_vm.fetch("env", {}) +create_vm_script = shell_without_comments(create_vm.fetch("run")) +assert!(!create_vm_env.key?("GH_TOKEN") && !create_vm_script.match?(/\bgh\s+api\b/), + "paid VM creation must consume only previously authorized source metadata") +assert!(create_vm_env["HBV4_VM_NAME"] == "${{ steps.identity.outputs.vm_name }}", + "Azure VM creation must use a task-specific HBV4_VM_NAME variable") +assert!(create_vm_script.include?('az vm create') && + create_vm_script.include?('--name "${HBV4_VM_NAME}"'), + "paid boundary must create the exact validated VM identity") +assert!(create_vm_script.include?("validate-created-hbv4-vm.sh"), + "paid boundary must validate the raw Azure VM response through the fixture-tested helper") +assert!(!create_vm_script.include?("diskSizeGb"), + "paid boundary must not use Azure's incorrectly cased disk-size field") +assert!(create_vm_script.include?('.storageProfile.osDisk.managedDisk.id') && + create_vm_script.include?('.networkProfile.networkInterfaces[0].id'), + "paid boundary must recover child resource IDs from the validated raw response") + +jit = find_one_step(provision_steps, "JIT configuration step") { |step| step["id"] == "jit" } +jit_script = shell_without_comments(jit.fetch("run")) +assert!(jit.fetch("env", {})["HBV4_JIT_NAME"] == "${{ steps.identity.outputs.runner_name }}", + "JIT configuration must use a semantic runner-name variable") +assert!(jit_script.include?("runner_id=") && jit_script.include?("runner.name == $name") && + jit_script.include?("labels[]"), + "JIT response must bind runner ID, exact name, and exact label") +assert!(provision.fetch("outputs", {}).fetch("runner_id", {})["value"] == + "${{ steps.jit.outputs.runner_id }}", + "cleanup identity must be captured before Azure VM creation can fail") + +render = find_one_step(provision_steps, "bootstrap render step") { |step| step["id"] == "bootstrap" } +render_env = render.fetch("env", {}) +render_script = shell_without_comments(render.fetch("run")) +assert!(render_env["BOOTSTRAP_IDENTITY"] == "${{ steps.identity.outputs.vm_name }}", + "bootstrap output naming must avoid protected RUNNER_NAME") +assert!(render_env["RUNNER_STORAGE_PROFILE"] == "hbv4-nvme-raid0" || + render_script.include?("export RUNNER_STORAGE_PROFILE='hbv4-nvme-raid0'"), + "HBv4 bootstrap rendering must select the reviewed RAID-0 storage profile") +assert!(render_script.include?("render-runner-bootstrap.py"), + "provision action must call the reviewed renderer") + +# Workload validation must prove a writable child of the RAID, never require +# the intentionally root-owned mount root itself to be world-writable. +builder_validation = find_one_step(steps_for(jobs.fetch("build_candidate")), + "builder host validation step") do |step| + step["name"] == "Validate fixed HBv4 host contract" +end +builder_validation_script = shell_without_comments(builder_validation.fetch("run")) +assert!(builder_validation_script.include?('--writable-path "${GITHUB_WORKSPACE}"'), + "builder host validation must probe its runner-owned RAID workspace") + +tester_validation = find_one_step(steps_for(jobs.fetch("test_candidate")), + "tester candidate validation step") do |step| + step["name"] == "Pull exact digest and run network-disabled validation" +end +tester_validation_script = shell_without_comments(tester_validation.fetch("run")) +assert!(tester_validation_script.include?('--writable-path "${GITHUB_WORKSPACE}"'), + "tester host validation must probe its runner-owned RAID workspace") +assert!(tester_validation_script.include?('validation_scratch="${TMPDIR}/container-validation-'), + "tester container scratch must remain below the writable RAID temporary directory") +assert!(!tester_validation_script.include?('validation_scratch="/mnt/hbv4-local/container-validation-'), + "tester must not create scratch directly under the protected RAID mount root") + +# The workflow declares its root spec alongside its other build configuration; +# provider manifests own the Zen 4 target and native compiler constraints. +workflow_env = workflow.fetch("env", {}) +assert!(workflow_env["SPEC"] == "~pygeosx ~docs %gcc-13", + "HBv4 workflow must declare the root spec with terminal %gcc-13") + +# Source mirroring removes the paid build's dependency on dozens of upstream +# project hosts without permitting binary reuse or an installed upstream. +provider_manifest_paths.each do |provider, path| + spack_config = load_yaml(path).fetch("spack") + mirrors = spack_config.fetch("mirrors:") + assert!(mirrors.keys == ["spack-public"], + "#{provider} must override lower scopes with only the reviewed source mirror") + public_mirror = mirrors.fetch("spack-public") + assert!(public_mirror["url"] == "https://mirror.spack.io" && + public_mirror["source"] == true && public_mirror["binary"] == false, + "#{provider} public mirror must be source-only") + assert!(spack_config.fetch("upstreams:").empty?, + "#{provider} must not reuse an installed Spack upstream") +end + +builder_steps = steps_for(jobs.fetch("build_candidate")) +candidate_build = find_one_step(builder_steps, "candidate image build step") do |step| + step["name"] == "Build without cache" +end +candidate_build_script = shell_without_comments(candidate_build.fetch("run")) +assert!(candidate_build_script.include?('--build-arg "SPEC=${SPEC}"'), + "candidate build must forward the declarative HBv4 root spec") +assert!(!candidate_build_script.include?("hbv4_spec="), + "candidate build must not hide the root spec in shell-local state") + +# A successful immutable promotion must surface a copy/pasteable digest-pinned +# pull command both in the job summary and in the retained lifecycle artifact. +finalizer_steps = steps_for(jobs.fetch("finalize_candidate")) +lifecycle_evidence = find_one_step(finalizer_steps, "lifecycle evidence step") do |step| + step["name"] == "Write sanitized lifecycle evidence" +end +lifecycle_script = shell_without_comments(lifecycle_evidence.fetch("run")) +assert!(lifecycle_script.include?('if [[ "${PROMOTE_OUTCOME}" == success ]]') && + lifecycle_script.include?('[[ "${final_digest}" =~ ^sha256:[0-9a-f]{64}$ ]]'), + "pull instructions must be emitted only for a valid promoted digest") +assert!(lifecycle_script.include?( + 'final_reference="${final_acr_registry}/${final_repository}@${final_digest}"' + ) && lifecycle_script.include?("docker pull %s"), + "pull instructions must use the immutable registry/repository@digest reference") +assert!(lifecycle_script.include?('${evidence_dir}/pull-instructions.md') && + lifecycle_script.include?('${GITHUB_STEP_SUMMARY}'), + "digest-pinned pull instructions must reach the artifact and job summary") + +lifecycle_upload = find_one_step(finalizer_steps, "lifecycle evidence upload step") do |step| + step["name"] == "Upload sanitized lifecycle evidence" +end +assert!(lifecycle_upload.fetch("with", {})["path"] == + "${{ runner.temp }}/hbv4-lifecycle-evidence", + "lifecycle artifact must retain the generated pull instructions") + +cleanup_steps = action_steps(cleanup) +cleanup_steps.each do |step| + environment = step.fetch("env", {}) + next unless environment.key?("GH_TOKEN") + + assert!(environment["GH_TOKEN"] == "${{ steps.runner_app_token.outputs.token }}", + "cleanup GH_TOKEN must be the runner-administration App token") + script = shell_without_comments(step.fetch("run", "")) + endpoints = script.scan(%r{repos/\$\{GITHUB_REPOSITORY\}/[^"'[:space:]]+}) + assert!(!endpoints.empty? && + endpoints.all? { |endpoint| endpoint.start_with?("repos/${GITHUB_REPOSITORY}/actions/runners") }, + "cleanup App token may call only repository runner endpoints") +end +cleanup_text = cleanup_steps.map { |step| shell_without_comments(step.fetch("run", "")) }.join("\n") +assert!(cleanup_text.include?("expected_label") && cleanup_text.include?(".labels[]?") && + (cleanup_text.include?(".name == $label") || cleanup_text.include?(". == $label")), + "cleanup must recover a pre-VM registration through its exact lifecycle label") + +bootstrap = File.read(bootstrap_path) +assert!(!bootstrap.match?(/^RUNNER_NAME=/) && !bootstrap.include?("@@RUNNER_NAME@@"), + "guest bootstrap must not define or render GitHub's protected RUNNER_NAME") +assert!(bootstrap.include?('sudo -u "${RUNNER_USER}" -H "${TRUSTED_VALIDATION_ROOT}/validate-host.sh"') && + bootstrap.include?('--writable-path "${RUNNER_HOME}"'), + "guest bootstrap must validate storage as the runner user in its RAID-backed home") + +host_validator = File.read(host_validator_path) +assert!(host_validator.include?("--writable-path") && + host_validator.include?('mktemp "$writable_path/.hbv4-write-test.XXXXXX"'), + "host validator must probe an explicit writable workload path") +assert!(!host_validator.include?('mktemp "$mount_path/.hbv4-write-test.XXXXXX"'), + "host validator must not require write permission on the protected RAID mount root") + +puts "HBv4 workflow/token/name contract passed." +RUBY diff --git a/ci/azure/tests/test-render-runner-bootstrap.py b/ci/azure/tests/test-render-runner-bootstrap.py new file mode 100755 index 00000000..ca984e13 --- /dev/null +++ b/ci/azure/tests/test-render-runner-bootstrap.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Offline contract tests for the reviewed Azure custom-data renderer.""" + +from __future__ import annotations + +import base64 +import gzip +import hashlib +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import unittest + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +RENDERER = REPOSITORY_ROOT / "ci/azure/scripts/render-runner-bootstrap.py" +BOOTSTRAP = REPOSITORY_ROOT / "ci/azure/scripts/runner-bootstrap.sh" +STORAGE_HELPER = REPOSITORY_ROOT / "ci/azure/scripts/setup-hbv4-local-nvme.sh" +HOST_VALIDATOR = REPOSITORY_ROOT / "scripts/hbv4/validate-host.sh" +CONTAINER_VALIDATOR = REPOSITORY_ROOT / "scripts/hbv4/validate-hbv4-tpls" +PARALLEL_HDF5_SOURCE = REPOSITORY_ROOT / "scripts/hbv4/parallel_hdf5_shared.c" +AZURE_CUSTOM_DATA_LIMIT = 65_535 + + +class RunnerBootstrapRendererContract(unittest.TestCase): + maxDiff = None + + def setUp(self) -> None: + self.helper_payload = STORAGE_HELPER.read_bytes() + self.helper_digest = hashlib.sha256(self.helper_payload).hexdigest() + self.helper_gzip_base64 = base64.b64encode( + gzip.compress(self.helper_payload, compresslevel=9, mtime=0) + ).decode("ascii") + + def encoded_file(path: Path) -> tuple[str, str]: + payload = path.read_bytes() + digest = hashlib.sha256(payload).hexdigest() + encoded = base64.b64encode( + gzip.compress(payload, compresslevel=9, mtime=0) + ).decode("ascii") + return digest, encoded + + host_digest, host_encoded = encoded_file(HOST_VALIDATOR) + container_digest, container_encoded = encoded_file(CONTAINER_VALIDATOR) + parallel_digest, parallel_encoded = encoded_file(PARALLEL_HDF5_SOURCE) + self.environment = os.environ.copy() + self.environment.update( + { + "RUNNER_ENCODED_JITCONFIG": "Zml4dHVyZS1qaXQtY29uZmln", + "RUNNER_VERSION": "2.335.1", + "RUNNER_TARBALL_SHA256": "a" * 64, + "ACR_NAME": "geoscifixture", + "RUNNER_VM_UAMI_CLIENT_ID": "00000000-0000-4000-8000-000000000001", + "RUNNER_STORAGE_PROFILE": "hbv4-nvme-raid0", + "STORAGE_HELPER_SHA256": self.helper_digest, + "STORAGE_HELPER_GZIP_BASE64": self.helper_gzip_base64, + "HOST_VALIDATOR_SHA256": host_digest, + "HOST_VALIDATOR_GZIP_BASE64": host_encoded, + "CONTAINER_VALIDATOR_SHA256": container_digest, + "CONTAINER_VALIDATOR_GZIP_BASE64": container_encoded, + "PARALLEL_HDF5_SOURCE_SHA256": parallel_digest, + "PARALLEL_HDF5_SOURCE_GZIP_BASE64": parallel_encoded, + } + ) + + def render( + self, + *, + environment: dict[str, str] | None = None, + source: Path = BOOTSTRAP, + ) -> tuple[subprocess.CompletedProcess[str], bytes]: + with tempfile.TemporaryDirectory() as temporary_directory: + output = Path(temporary_directory) / "custom-data.sh" + completed = subprocess.run( + [sys.executable, str(RENDERER), str(source), str(output)], + check=False, + env=environment or self.environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + payload = output.read_bytes() if output.exists() else b"" + return completed, payload + + @staticmethod + def assignment(payload: str, variable: str) -> str: + match = re.search( + rf'^{re.escape(variable)}="([^"]*)"$', payload, flags=re.MULTILINE + ) + if match is None: + raise AssertionError(f"rendered payload omitted {variable}") + return match.group(1) + + @staticmethod + def unwrap(wrapper: bytes) -> tuple[bytes, str]: + wrapper_text = wrapper.decode("ascii") + digest_match = re.search( + r"^readonly PAYLOAD_SHA256='([0-9a-f]{64})'$", + wrapper_text, + flags=re.MULTILINE, + ) + if digest_match is None: + raise AssertionError("custom-data wrapper omitted PAYLOAD_SHA256") + payload_match = re.search( + r"^if ! base64 --decode <<'__GEOS_HBV4_BOOTSTRAP__' \| gzip -dc > " + r'"\$\{payload_path\}"\n(?P[A-Za-z0-9+/=\n]+)' + r"^__GEOS_HBV4_BOOTSTRAP__$", + wrapper_text, + flags=re.MULTILINE, + ) + if payload_match is None: + raise AssertionError("custom-data wrapper omitted its encoded payload") + encoded = "".join(payload_match.group("encoded").splitlines()) + payload = gzip.decompress(base64.b64decode(encoded, validate=True)) + return payload, digest_match.group(1) + + def test_renders_ascii_payload_and_embeds_exact_storage_helper(self) -> None: + completed, rendered_bytes = self.render() + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertGreater(len(rendered_bytes), 0) + self.assertLessEqual(len(rendered_bytes), AZURE_CUSTOM_DATA_LIMIT) + + wrapper = rendered_bytes.decode("ascii") + self.assertIn("base64 --decode", wrapper) + self.assertIn("gzip -dc", wrapper) + self.assertIn('"${payload_path}"', wrapper) + self.assertNotIn('exec "${payload_path}"', wrapper) + + inner_bytes, wrapper_digest = self.unwrap(rendered_bytes) + self.assertEqual(hashlib.sha256(inner_bytes).hexdigest(), wrapper_digest) + self.assertGreater(len(inner_bytes), 0) + rendered = inner_bytes.decode("ascii") + self.assertNotRegex(wrapper, r"@@[A-Z0-9_]+@@") + self.assertNotRegex(rendered, r"@@[A-Z0-9_]+@@") + self.assertEqual( + self.assignment(rendered, "RUNNER_STORAGE_PROFILE"), + "hbv4-nvme-raid0", + ) + self.assertEqual( + self.assignment(rendered, "RUNNER_ENCODED_JITCONFIG"), + self.environment["RUNNER_ENCODED_JITCONFIG"], + ) + self.assertIn("download_with_retries() {", rendered) + self.assertIn("for attempt in 1 2 3 4 5; do", rendered) + self.assertIn("--connect-timeout 20", rendered) + self.assertIn("--max-time 180", rendered) + self.assertIn("curl ca-certificates jq gh git git-lfs", rendered) + self.assertIn( + '"${tmp}/${tarball}" \\\n "GitHub Actions runner tarball"', + rendered, + ) + + encoded_helper = self.assignment(rendered, "STORAGE_HELPER_GZIP_BASE64") + embedded_digest = self.assignment(rendered, "STORAGE_HELPER_SHA256") + decoded_helper = gzip.decompress(base64.b64decode(encoded_helper, validate=True)) + decoded_digest = hashlib.sha256(decoded_helper).hexdigest() + self.assertEqual(decoded_helper, self.helper_payload) + self.assertEqual(len(decoded_helper), len(self.helper_payload)) + self.assertEqual(embedded_digest, self.helper_digest) + self.assertEqual(decoded_digest, embedded_digest) + + def test_missing_required_value_fails_without_output(self) -> None: + environment = self.environment.copy() + environment.pop("RUNNER_STORAGE_PROFILE") + completed, rendered = self.render(environment=environment) + self.assertNotEqual(completed.returncode, 0) + self.assertEqual(rendered, b"") + self.assertIn("RUNNER_STORAGE_PROFILE", completed.stderr) + + def test_multiline_value_is_rejected(self) -> None: + environment = self.environment.copy() + environment["ACR_NAME"] = "registry\nsecond-line" + completed, rendered = self.render(environment=environment) + self.assertNotEqual(completed.returncode, 0) + self.assertEqual(rendered, b"") + self.assertIn("not single-line", completed.stderr) + + def test_duplicate_marker_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + source = Path(temporary_directory) / "duplicate-marker.sh" + source.write_text( + BOOTSTRAP.read_text(encoding="utf-8") + "\n@@ACR_NAME@@\n", + encoding="utf-8", + ) + completed, rendered = self.render(source=source) + self.assertNotEqual(completed.returncode, 0) + self.assertEqual(rendered, b"") + self.assertIn("@@ACR_NAME@@ must occur exactly once", completed.stderr) + + def test_unknown_marker_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + source = Path(temporary_directory) / "unknown-marker.sh" + source.write_text( + BOOTSTRAP.read_text(encoding="utf-8") + "\n@@UNREVIEWED_VALUE@@\n", + encoding="utf-8", + ) + completed, rendered = self.render(source=source) + self.assertNotEqual(completed.returncode, 0) + self.assertEqual(rendered, b"") + self.assertIn("unrecognized template placeholder", completed.stderr) + + def test_payload_larger_than_azure_limit_is_rejected(self) -> None: + environment = self.environment.copy() + # Incompressible base64 models a large real JIT payload; a repeated + # character would compress below Azure's custom-data limit. + environment["RUNNER_ENCODED_JITCONFIG"] = base64.b64encode( + os.urandom(90_000) + ).decode("ascii") + completed, rendered = self.render(environment=environment) + self.assertNotEqual(completed.returncode, 0) + self.assertEqual(rendered, b"") + self.assertIn("Azure custom data", completed.stderr) + self.assertIn("limit is 65535", completed.stderr) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/ci/azure/tests/test-runner-bootstrap-probe.sh b/ci/azure/tests/test-runner-bootstrap-probe.sh new file mode 100755 index 00000000..bca90b1e --- /dev/null +++ b/ci/azure/tests/test-runner-bootstrap-probe.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash + +# @file test-runner-bootstrap-probe.sh +# +# Credential-free fixtures for the guest bootstrap-state probe. +# +# Each case redirects the production probe into an isolated file tree and +# PATH-injected Azure guest-command shims. Positive fixtures model a progressing +# bootstrap; negative fixtures model one terminal or malformed state at a time. +# The assertions protect the precedence invariants: a validated status record +# takes precedence over cloud-final's coarser fallback; cloud-final failure takes +# precedence over a stale running record, including runner-agent handoff; +# terminal state is detected without waiting for GitHub's outer timeout; +# runner-process handoff and the bounded inconclusive state are distinct; and no +# credential-bearing guest source is ever read. Terminal machine markers must +# remain the final output line. Race fixtures also replace a running generation +# while cloud-final is inspected, proving the probe re-reads once and retains +# the EXIT trap's more precise phase and exit code. + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +probe="${repo_root}/ci/azure/scripts/probe-runner-bootstrap.sh" +tmp_dir="$(mktemp -d)" +trap 'rm -rf "${tmp_dir}"' EXIT + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +make_mock() +{ + local name="$1" + shift + { + echo "#!/usr/bin/env bash" + echo "set -euo pipefail" + printf "%s\n" "$@" + } > "${tmp_dir}/bin/${name}" + chmod +x "${tmp_dir}/bin/${name}" +} + +mkdir -p \ + "${tmp_dir}/bin" \ + "${tmp_dir}/root/run" \ + "${tmp_dir}/root/var/log" \ + "${tmp_dir}/root/var/lib/waagent" \ + "${tmp_dir}/root/proc" \ + "${tmp_dir}/root/mnt/hbv4-local" \ + "${tmp_dir}/root/runner/_diag" + +# These mock bodies are written verbatim into child scripts; their parameter +# expansions must occur when the probe invokes the child, not while the fixture +# creates it. +# shellcheck disable=SC2016 +make_mock cloud-init \ + '[[ "${1:-}" == "status" ]] || exit 2' \ + 'printf "%s\n" "${MOCK_CLOUD_INIT_STATUS:-status: running}"' +# shellcheck disable=SC2016 +make_mock systemctl \ + 'if [[ "${1:-}" == "is-failed" ]]; then' \ + ' if [[ "${MOCK_CREATE_STATUS_DURING_CLOUD_FINAL_CHECK:-0}" == "1" ]]; then' \ + ' printf "version=1\nstate=failed\nphase=install-azure-cli\nexit_code=100\nupdated_at=2026-07-27T22:00:00Z\n" > "${MOCK_STATUS_FILE:?}"' \ + ' fi' \ + ' [[ "${MOCK_CLOUD_FINAL_FAILED:-0}" == "1" ]]' \ + ' exit' \ + 'fi' \ + 'if [[ "${1:-}" == "show" ]]; then echo "Id=docker.service"; echo "ActiveState=inactive"; echo "Result=success"; exit 0; fi' \ + 'exit 1' +make_mock lsblk 'echo "nvme0n1 disk 1.7T Microsoft NVMe Direct Disk"' +make_mock findmnt 'exit 1' + +printf "PersonalData=DO_NOT_READ_CUSTOM_DATA\n" \ + > "${tmp_dir}/root/var/lib/waagent/CustomData" +printf "DO_NOT_READ_RUNNER_DIAGNOSTICS\n" \ + > "${tmp_dir}/root/runner/_diag/Runner_test.log" +printf "ERROR: DO_NOT_READ_ARBITRARY_CLOUD_INIT_OUTPUT\n" \ + > "${tmp_dir}/root/var/log/cloud-init-output.log" +printf "unused\n" > "${tmp_dir}/root/proc/mdstat" +printf "safe bootstrap phase line\n" \ + > "${tmp_dir}/root/var/log/geos-runner-bootstrap.log" + +write_status() +{ + local state="$1" + local phase="$2" + local exit_code="$3" + { + echo "version=1" + echo "state=${state}" + echo "phase=${phase}" + echo "exit_code=${exit_code}" + echo "updated_at=2026-07-27T22:00:00Z" + } > "${tmp_dir}/root/run/geos-runner-bootstrap.status" +} + +run_probe() +{ + PATH="${tmp_dir}/bin:${PATH}" \ + GEOS_BOOTSTRAP_PROBE_TEST_MODE=1 \ + GEOS_BOOTSTRAP_PROBE_TEST_ROOT="${tmp_dir}/root" \ + GEOS_BOOTSTRAP_PROBE_TEST_TIMEOUT_SECONDS=1 \ + GEOS_BOOTSTRAP_PROBE_TEST_POLL_SECONDS=1 \ + MOCK_CLOUD_INIT_STATUS="${MOCK_CLOUD_INIT_STATUS:-status: running}" \ + MOCK_CLOUD_FINAL_FAILED="${MOCK_CLOUD_FINAL_FAILED:-0}" \ + MOCK_CREATE_STATUS_DURING_CLOUD_FINAL_CHECK="${MOCK_CREATE_STATUS_DURING_CLOUD_FINAL_CHECK:-0}" \ + MOCK_STATUS_FILE="${tmp_dir}/root/run/geos-runner-bootstrap.status" \ + "${probe}" +} + +assert_no_sensitive_source() +{ + local output="$1" + [[ "${output}" != *"DO_NOT_READ_CUSTOM_DATA"* ]] || + fail "probe read Azure custom data" + [[ "${output}" != *"DO_NOT_READ_RUNNER_DIAGNOSTICS"* ]] || + fail "probe read runner diagnostics" + [[ "${output}" != *"DO_NOT_READ_ARBITRARY_CLOUD_INIT_OUTPUT"* ]] || + fail "probe read arbitrary cloud-init output" +} + +write_status running configure-runner-storage 0 +output="$(run_probe)" +[[ "${output}" == "GEOS_BOOTSTRAP_STATE=inconclusive last_state=running phase=configure-runner-storage updated_at=2026-07-27T22:00:00Z" ]] || + fail "bounded progressing status was not reported as inconclusive" + +write_status running runner-agent 0 +output="$(run_probe)" +[[ "${output}" == "GEOS_BOOTSTRAP_STATE=handoff phase=runner-agent updated_at=2026-07-27T22:00:00Z" ]] || + fail "live runner-process handoff was not reported exactly" + +# A SIGKILL/OOM termination cannot run the bootstrap's EXIT trap, so the last +# valid running generation can outlive its custom-data supervisor. Once +# cloud-final is failed, that record is stale and must not consume the remainder +# of the paid VM's outer health-check allowance. +write_status running configure-runner-storage 0 +MOCK_CLOUD_FINAL_FAILED=1 +output="$(run_probe)" +unset MOCK_CLOUD_FINAL_FAILED +[[ "$(tail -n 1 <<< "${output}")" == \ + "GEOS_BOOTSTRAP_STATE=terminal source=cloud-init phase=cloud-final exit_code=1" ]] || + fail "failed cloud-final did not override a stale running bootstrap phase" +[[ "${output}" != *"GEOS_BOOTSTRAP_STATE=handoff"* ]] || + fail "stale running bootstrap phase was reported as handoff" +[[ "${#output}" -lt 4096 ]] || fail "stale-running diagnostics exceeded Azure output headroom" +assert_no_sensitive_source "${output}" + +# runner-agent proves only that the child survived its five-second launch guard. +# It is a valid handoff while cloud-final is active, but not after the supervisor +# has terminated and can no longer reap or report the child. +write_status running runner-agent 0 +MOCK_CLOUD_FINAL_FAILED=1 +output="$(run_probe)" +unset MOCK_CLOUD_FINAL_FAILED +[[ "$(tail -n 1 <<< "${output}")" == \ + "GEOS_BOOTSTRAP_STATE=terminal source=cloud-init phase=cloud-final exit_code=1" ]] || + fail "failed cloud-final did not override stale runner-agent handoff" +[[ "${output}" != *"GEOS_BOOTSTRAP_STATE=handoff"* ]] || + fail "stale runner-agent state was reported as handoff" +assert_no_sensitive_source "${output}" + +# Model the normal race in which the EXIT trap replaces a running generation +# while the probe observes cloud-final becoming failed. The re-read must prefer +# the precise status record rather than emitting cloud-final's generic marker. +write_status running install-azure-cli 0 +MOCK_CLOUD_FINAL_FAILED=1 +MOCK_CREATE_STATUS_DURING_CLOUD_FINAL_CHECK=1 +output="$(run_probe)" +unset MOCK_CLOUD_FINAL_FAILED MOCK_CREATE_STATUS_DURING_CLOUD_FINAL_CHECK +[[ "$(tail -n 1 <<< "${output}")" == \ + "GEOS_BOOTSTRAP_STATE=terminal source=status-file phase=install-azure-cli exit_code=100" ]] || + fail "terminal status published over a running record did not win the cloud-final race" +assert_no_sensitive_source "${output}" + +write_status failed configure-runner-storage 1 +output="$(run_probe)" +[[ "$(tail -n 1 <<< "${output}")" == \ + "GEOS_BOOTSTRAP_STATE=terminal source=status-file phase=configure-runner-storage exit_code=1" ]] || + fail "failed status did not end with the terminal marker" +[[ "${#output}" -lt 4096 ]] || fail "terminal diagnostics exceeded Azure output headroom" +assert_no_sensitive_source "${output}" + +# Once bootstrap has published a valid terminal record, cloud-final may also be +# failed but is no longer authoritative. Preserve the bootstrap's precise phase +# and exit code so the control job reports the actionable guest failure. +write_status failed install-azure-cli 100 +MOCK_CLOUD_FINAL_FAILED=1 +output="$(run_probe)" +unset MOCK_CLOUD_FINAL_FAILED +[[ "$(tail -n 1 <<< "${output}")" == \ + "GEOS_BOOTSTRAP_STATE=terminal source=status-file phase=install-azure-cli exit_code=100" ]] || + fail "cloud-final failure masked the valid bootstrap terminal status" +assert_no_sensitive_source "${output}" + +write_status exited runner-agent 0 +output="$(run_probe)" +[[ "$(tail -n 1 <<< "${output}")" == \ + "GEOS_BOOTSTRAP_STATE=terminal source=status-file phase=runner-agent exit_code=0" ]] || + fail "early zero-status runner exit was not terminal" + +write_status running runner-agent 9 +output="$(run_probe)" +[[ "$(tail -n 1 <<< "${output}")" == *"phase=invalid-running-exit-code exit_code=1" ]] || + fail "contradictory running state was not rejected" + +printf "version=1\nstate=running\n" \ + > "${tmp_dir}/root/run/geos-runner-bootstrap.status" +output="$(run_probe)" +[[ "$(tail -n 1 <<< "${output}")" == *"phase=invalid-record exit_code=1" ]] || + fail "truncated status record was not rejected" + +write_status "DO_NOT_ECHO_MALFORMED_STATUS" runner-agent 0 +output="$(run_probe)" +[[ "$(tail -n 1 <<< "${output}")" == *"phase=invalid-value exit_code=1" ]] || + fail "unallowlisted status value was not rejected" +[[ "${output}" != *"DO_NOT_ECHO_MALFORMED_STATUS"* ]] || + fail "probe echoed malformed status-file content" + +rm -f "${tmp_dir}/root/run/geos-runner-bootstrap.status" +output="$(run_probe)" +[[ "${output}" == "GEOS_BOOTSTRAP_STATE=inconclusive last_state=missing" ]] || + fail "bounded missing status did not preserve the slow-boot path" + +# Model the status file appearing while the fallback checks cloud-final. The +# probe must restart status parsing instead of emitting the coarser cloud-final +# marker observed in the original Azure CLI provisioning failure. +MOCK_CLOUD_FINAL_FAILED=1 +MOCK_CREATE_STATUS_DURING_CLOUD_FINAL_CHECK=1 +output="$(run_probe)" +unset MOCK_CLOUD_FINAL_FAILED MOCK_CREATE_STATUS_DURING_CLOUD_FINAL_CHECK +[[ "$(tail -n 1 <<< "${output}")" == \ + "GEOS_BOOTSTRAP_STATE=terminal source=status-file phase=install-azure-cli exit_code=100" ]] || + fail "status published during cloud-final inspection was not preferred" +assert_no_sensitive_source "${output}" + +rm -f "${tmp_dir}/root/run/geos-runner-bootstrap.status" +MOCK_CLOUD_INIT_STATUS="status: error" +output="$(run_probe)" +unset MOCK_CLOUD_INIT_STATUS +[[ "$(tail -n 1 <<< "${output}")" == \ + "GEOS_BOOTSTRAP_STATE=terminal source=cloud-init phase=cloud-final exit_code=1" ]] || + fail "cloud-init failure before status creation was not terminal" +assert_no_sensitive_source "${output}" + +echo "runner bootstrap probe fixtures passed." diff --git a/ci/azure/tests/test-runner-names.sh b/ci/azure/tests/test-runner-names.sh new file mode 100755 index 00000000..b69a7182 --- /dev/null +++ b/ci/azure/tests/test-runner-names.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +source "${repo_root}/ci/azure/scripts/runner-names.sh" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +identity="$(geos_ci_runner_identity 123456 2 'Ubuntu 24.04 - gcc 13 debug')" +grep -q '^runner_label=geos-hbv4-123456-2-ubuntu-24-04-gcc-13-debug$' <<< "${identity}" \ + || fail "unexpected runner label for normal matrix row" +grep -q '^vm_name=geos-hbv4-123456-2-ubuntu-24-04-gcc-13-debug$' <<< "${identity}" \ + || fail "unexpected VM name for normal matrix row" + +long_identity="$(geos_ci_runner_identity 123456789012345 12 'this is a deliberately long matrix row name that must fit azure vm names')" +long_vm_name="$(awk -F= '$1 == "vm_name" { print $2 }' <<< "${long_identity}")" +(( ${#long_vm_name} <= 64 )) || fail "VM name exceeds Azure's 64-character Linux VM name limit" + +# Long-name truncation must AVOID a hash-suffix collision. +# +# Malformed state being modeled: two distinct matrix instance keys that share a +# common prefix long enough that a naive 64-character truncation would collapse +# them to the SAME string. If the identity helper truncated without a +# distinguishing suffix, both legs would claim one VM name / runner label, and +# the janitor's runner_name==vm_name join +# (ci/azure/scripts/runner-names.sh, ci/azure/scripts/janitor.sh) +# would no longer identify a unique backing VM per registration. +# +# Invariant protected: distinct instance keys yield distinct vm_names, each still +# <= 64 chars. runner-names.sh enforces this by appending a deterministic hash of +# the FULL pre-truncation base (which includes the differing slug tail), so the +# hash differs even when the visible truncated prefix is identical. The two keys +# below share a 90-char run of 'a' and differ only in a trailing word, which is +# exactly the case that collides under plain truncation. +collide_prefix="$(printf 'a%.0s' $(seq 1 90))" +collide_key_a="${collide_prefix}-alpha" +collide_key_b="${collide_prefix}-omega" +collide_vm_a="$(geos_ci_runner_identity 1 1 "${collide_key_a}" | awk -F= '$1 == "vm_name" { print $2 }')" +collide_vm_b="$(geos_ci_runner_identity 1 1 "${collide_key_b}" | awk -F= '$1 == "vm_name" { print $2 }')" +(( ${#collide_vm_a} <= 64 )) || fail "truncated VM name (a) exceeds Azure's 64-character limit" +(( ${#collide_vm_b} <= 64 )) || fail "truncated VM name (b) exceeds Azure's 64-character limit" +[[ "${collide_vm_a}" != "${collide_vm_b}" ]] \ + || fail "long instance keys sharing a 64-char prefix collided to the same VM name" + +# Punctuation-only instance key must fall back to the 'job' slug. +# +# Malformed state being modeled: an instance key containing no alphanumeric +# characters at all (here '!!!---@@@'). geos_ci_slugify strips every non +# [a-z0-9] run, which would leave an EMPTY slug and produce a trailing-dash VM +# name like 'geos-hbv4-100-1-' that is both ugly and an invalid leaf. The helper +# guards this by substituting the literal slug 'job' when the slugified result is +# empty, so the name remains well-formed and join-stable. +punct_identity="$(geos_ci_runner_identity 100 1 '!!!---@@@')" +grep -q '^vm_name=geos-hbv4-100-1-job$' <<< "${punct_identity}" \ + || fail "punctuation-only instance key should fall back to the 'job' slug" + +if geos_ci_runner_identity abc 1 row >/dev/null 2>&1; then + fail "non-numeric run id should be rejected" +fi + +echo "runner name tests passed" diff --git a/docker/install-mpi4py.sh b/docker/install-mpi4py.sh new file mode 100755 index 00000000..8cf98706 --- /dev/null +++ b/docker/install-mpi4py.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -euo pipefail + +readonly MPI4PY_VERSION=4.1.2 +readonly MPI4PY_SHA256=56860286dc45f20e8821e93cb06669e30462348bf866f685553fa4b712d58d02 +readonly MPI4PY_ARCHIVE=mpi4py-${MPI4PY_VERSION}.tar.gz +readonly MPI4PY_URL=https://files.pythonhosted.org/packages/source/m/mpi4py/${MPI4PY_ARCHIVE} + +: "${MPICC:?MPICC must name the selected MPI C compiler wrapper}" +: "${MPI_HOME:?MPI_HOME must name the selected MPI prefix}" + +build_dir=$(mktemp -d /tmp/mpi4py-build.XXXXXX) +trap 'rm -rf "${build_dir}"' EXIT + +curl --fail --location --silent --show-error \ + --output "${build_dir}/${MPI4PY_ARCHIVE}" \ + "${MPI4PY_URL}" +echo "${MPI4PY_SHA256} ${build_dir}/${MPI4PY_ARCHIVE}" | sha256sum --check --strict + +MPICC="${MPICC}" python3 -m pip install \ + --break-system-packages \ + --no-cache-dir \ + --no-binary=mpi4py \ + "${build_dir}/${MPI4PY_ARCHIVE}" + +mpi4py_extension=$(python3 - <<'PY' +import mpi4py +from mpi4py import MPI + +assert mpi4py.__version__ == "4.1.2" +print(MPI.__file__) +print(MPI.Get_library_version().replace("\n", " ")) +PY +) +extension_path=${mpi4py_extension%%$'\n'*} +[[ -f "${extension_path}" ]] || { + echo "mpi4py extension is missing: ${extension_path}" >&2 + exit 1 +} +mapfile -t linked_mpi < <( + ldd "${extension_path}" | awk ' + /(^|[[:space:]])libmpi[^[:space:]]*/ { + for (i = 1; i <= NF; ++i) if ($i == "=>") print $(i + 1) + }' +) +(( ${#linked_mpi[@]} > 0 )) || { + echo 'mpi4py has no dynamic libmpi dependency' >&2 + exit 1 +} +for library in "${linked_mpi[@]}"; do + library=$(readlink -f "${library}") + [[ "${library}" == "${MPI_HOME}"/* ]] || { + echo "mpi4py resolves libmpi outside ${MPI_HOME}: ${library}" >&2 + exit 1 + } +done +printf '%s\n' "${mpi4py_extension#*$'\n'}" diff --git a/docker/install-mpich.sh b/docker/install-mpich.sh new file mode 100755 index 00000000..691577b0 --- /dev/null +++ b/docker/install-mpich.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -euo pipefail + +readonly MPICH_VERSION=5.0.1 +readonly MPICH_SHA256=8c1832a13ddacf071685069f5fadfd1f2877a29e1a628652892c65211b1f3327 +readonly MPICH_PREFIX=/opt/mpich/${MPICH_VERSION} +readonly MPICH_ARCHIVE=mpich-${MPICH_VERSION}.tar.gz +readonly MPICH_URL=https://www.mpich.org/static/downloads/${MPICH_VERSION}/${MPICH_ARCHIVE} + +configure_args=( + "--prefix=${MPICH_PREFIX}" + '--with-device=ch4:ofi' + '--with-libfabric=embedded' + '--enable-romio' + '--with-pm=hydra' + '--enable-shared' + '--disable-static' +) +if [[ -n "${HBV4_COMPILER_FLAGS:-}" ]]; then + export MPICH_MPICC_CFLAGS="${HBV4_COMPILER_FLAGS}" + export MPICH_MPICXX_CXXFLAGS="${HBV4_COMPILER_FLAGS}" + export MPICH_MPIF77_FFLAGS="${HBV4_COMPILER_FLAGS}" + export MPICH_MPIFORT_FCFLAGS="${HBV4_COMPILER_FLAGS}" +fi +{ + if [[ -n "${HBV4_COMPILER_FLAGS:-}" ]]; then + printf 'CFLAGS=%s\nCXXFLAGS=%s\nFCFLAGS=%s\nFFLAGS=%s\n' \ + "${CFLAGS}" "${CXXFLAGS}" "${FCFLAGS}" "${FFLAGS}" + printf 'MPICH_MPICC_CFLAGS=%s\nMPICH_MPICXX_CXXFLAGS=%s\nMPICH_MPIF77_FFLAGS=%s\nMPICH_MPIFORT_FCFLAGS=%s\n' \ + "${MPICH_MPICC_CFLAGS}" "${MPICH_MPICXX_CXXFLAGS}" \ + "${MPICH_MPIF77_FFLAGS}" "${MPICH_MPIFORT_FCFLAGS}" + fi + printf '%s\n' "${configure_args[@]}" +} > /opt/mpi-configure-args.txt + +build_dir=$(mktemp -d /tmp/mpich-build.XXXXXX) +trap 'rm -rf "${build_dir}"' EXIT + +curl --fail --location --silent --show-error \ + --output "${build_dir}/${MPICH_ARCHIVE}" \ + "${MPICH_URL}" +echo "${MPICH_SHA256} ${build_dir}/${MPICH_ARCHIVE}" | sha256sum --check --strict + +tar -xzf "${build_dir}/${MPICH_ARCHIVE}" -C "${build_dir}" +cd "${build_dir}/mpich-${MPICH_VERSION}" + +./configure "${configure_args[@]}" +make -j"$(nproc)" +make install + +"${MPICH_PREFIX}/bin/mpichversion" +"${MPICH_PREFIX}/bin/mpicc" -show diff --git a/docker/install-openmpi.sh b/docker/install-openmpi.sh new file mode 100755 index 00000000..4f1f2cb5 --- /dev/null +++ b/docker/install-openmpi.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -euo pipefail + +readonly OPENMPI_VERSION=5.0.10 +readonly OPENMPI_SHA256=0acecc4fc218e5debdbcb8a41d182c6b0f1d29393015ed763b2a91d5d7374cc6 +readonly OPENMPI_PREFIX=/opt/openmpi/${OPENMPI_VERSION} +readonly OPENMPI_ARCHIVE=openmpi-${OPENMPI_VERSION}.tar.bz2 +readonly OPENMPI_URL=https://download.open-mpi.org/release/open-mpi/v5.0/${OPENMPI_ARCHIVE} + +configure_args=( + "--prefix=${OPENMPI_PREFIX}" + '--enable-shared' + '--disable-static' + '--enable-mpi-fortran=all' + '--enable-io-romio' +) +if [[ -n "${HBV4_COMPILER_FLAGS:-}" ]]; then + configure_args+=( + "--with-wrapper-cflags=${HBV4_COMPILER_FLAGS}" + "--with-wrapper-cxxflags=${HBV4_COMPILER_FLAGS}" + "--with-wrapper-fcflags=${HBV4_COMPILER_FLAGS}" + ) +fi +{ + if [[ -n "${HBV4_COMPILER_FLAGS:-}" ]]; then + printf 'CFLAGS=%s\nCXXFLAGS=%s\nFCFLAGS=%s\nFFLAGS=%s\n' \ + "${CFLAGS}" "${CXXFLAGS}" "${FCFLAGS}" "${FFLAGS}" + fi + printf '%s\n' "${configure_args[@]}" +} > /opt/mpi-configure-args.txt + +build_dir=$(mktemp -d /tmp/openmpi-build.XXXXXX) +trap 'rm -rf "${build_dir}"' EXIT + +curl --fail --location --silent --show-error \ + --output "${build_dir}/${OPENMPI_ARCHIVE}" \ + "${OPENMPI_URL}" +echo "${OPENMPI_SHA256} ${build_dir}/${OPENMPI_ARCHIVE}" | sha256sum --check --strict + +tar -xjf "${build_dir}/${OPENMPI_ARCHIVE}" -C "${build_dir}" +cd "${build_dir}/openmpi-${OPENMPI_VERSION}" + +./configure "${configure_args[@]}" +make -j"$(nproc)" +make install + +# Both MPI-IO implementations are built; select OMPIO unless a caller explicitly +# overrides the MCA parameter at runtime. +printf '%s\n' 'io = ompio' >> "${OPENMPI_PREFIX}/etc/openmpi-mca-params.conf" + +ompi_io_components=$("${OPENMPI_PREFIX}/bin/ompi_info" --param io all) +grep -q 'MCA io: ompio' <<<"${ompi_io_components}" +grep -q 'MCA io: romio341' <<<"${ompi_io_components}" +"${OPENMPI_PREFIX}/bin/mpicc" --showme:command diff --git a/docker/tpl-ubuntu.Dockerfile b/docker/tpl-ubuntu.Dockerfile index 367092a3..c4d86638 100644 --- a/docker/tpl-ubuntu.Dockerfile +++ b/docker/tpl-ubuntu.Dockerfile @@ -17,28 +17,62 @@ ARG TMP_DIR=/tmp ARG SRC_DIR=$TMP_DIR/thirdPartyLibs ARG BLD_DIR=$TMP_DIR/build +# Portable builds default to Open MPI. HBv4 builds select one of the two +# provider-specific manifests below; all values are validated in the image. +ARG MPI_PROVIDER=openmpi +ARG MPI_PREFIX=/opt/openmpi/5.0.10 +ARG MPIEXEC_PATH=/opt/openmpi/5.0.10/bin/mpirun +ARG SPACK_ENV_FILE=docker/ubuntu-spack.yaml +ARG HBV4_BUILD=0 +ARG SOURCE_SHA +ARG FINAL_IMAGE_VARIANT=portable + ARG DOCKER_BASE_IMAGE=ubuntu:24.04 FROM ${DOCKER_BASE_IMAGE} AS tpl_toolchain_intersect_geosx_toolchain ARG SRC_DIR ARG CLANG_VERSION +ARG DOCKER_BASE_IMAGE +ARG MPI_PROVIDER +ARG MPI_PREFIX +ARG MPIEXEC_PATH +ARG SPACK_ENV_FILE +ARG HBV4_BUILD +ARG SOURCE_SHA +ARG FINAL_IMAGE_VARIANT # Install directory provided as a docker build argument; forwarded via ENV # (GEOSX_TPL_DIR is part of the image contract consumed by GEOS). ARG INSTALL_DIR ENV GEOSX_TPL_DIR=$INSTALL_DIR +# Fail before downloading or compiling anything if an unrecognized provider, +# manifest, or HBv4 metadata tuple was supplied. This also prevents build-arg +# paths from escaping the checked-in manifest allowlist. +RUN case "${MPI_PROVIDER}:${MPI_PREFIX}:${MPIEXEC_PATH}:${SPACK_ENV_FILE}:${HBV4_BUILD}:${FINAL_IMAGE_VARIANT}" in \ + openmpi:/opt/openmpi/5.0.10:/opt/openmpi/5.0.10/bin/mpirun:docker/ubuntu-spack.yaml:0:portable|\ + openmpi:/opt/openmpi/5.0.10:/opt/openmpi/5.0.10/bin/mpirun:docker/ubuntu-hbv4-openmpi-spack.yaml:1:hbv4|\ + mpich:/opt/mpich/5.0.1:/opt/mpich/5.0.1/bin/mpiexec:docker/ubuntu-hbv4-mpich-spack.yaml:1:hbv4) ;; \ + *) echo "ERROR: invalid MPI_PROVIDER/SPACK_ENV_FILE/HBV4_BUILD selection" >&2; exit 1 ;; \ + esac && \ + if [ "${HBV4_BUILD}" = 1 ]; then \ + printf '%s\n' "${SOURCE_SHA}" | grep -Eq '^[0-9a-f]{40}$' || \ + { echo "ERROR: SOURCE_SHA must be a 40-digit lowercase hexadecimal commit for HBv4 builds" >&2; exit 1; } && \ + printf '%s\n' "${DOCKER_BASE_IMAGE}" | grep -Eq '^.+@sha256:[0-9a-f]{64}$' || \ + { echo "ERROR: HBv4 DOCKER_BASE_IMAGE must be pinned by sha256 digest" >&2; exit 1; } ; \ + fi + # Packages needed both for the TPL build and for the downstream GEOS build. # We avoid reinstalling anything already present in the base image (compiler, # cmake, doxygen, blas/lapack-dev when included by base PACKAGES, etc.). RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ apt-get install -y --no-install-recommends \ + bzip2 \ ca-certificates \ + curl \ libtbb12 \ libgfortran5 \ zlib1g-dev \ - openmpi-bin \ - libopenmpi-dev \ python3-pip \ python3-sphinx \ python3-dev \ @@ -49,7 +83,6 @@ RUN apt-get update && \ unzip \ libmpfr-dev \ lbzip2 \ - bzip2 \ gnupg && \ apt-get clean && rm -rf /var/lib/apt/lists/* @@ -57,16 +90,9 @@ RUN apt-get update && \ # place; Ubuntu 24.04's pip package cannot be uninstalled by pip. RUN python3 -m pip install --break-system-packages clingo -# MPI environment. CC/CXX/FC come from the base image. -ENV MPICC=/usr/bin/mpicc \ - MPICXX=/usr/bin/mpicxx \ - MPIEXEC=/usr/bin/mpirun -ENV OMPI_CC=${CC} \ - OMPI_CXX=${CXX} - # For clang-based base images: -# 1) install a matching OpenMP runtime (libomp) -# 2) retarget OpenMPI wrappers to clang/clang++ +# install a matching OpenMP runtime (libomp). MPI itself is compiled with +# the CC/CXX/FC contract supplied by the pinned base image. RUN if echo "${CC}" | grep -q "clang"; then \ CLANG_MAJOR="${CLANG_VERSION:-}" ; \ if [ -z "${CLANG_MAJOR}" ]; then \ @@ -85,22 +111,61 @@ RUN if echo "${CC}" | grep -q "clang"; then \ DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ apt-get install -y --no-install-recommends libomp-dev ; \ fi ; \ - apt-get clean && rm -rf /var/lib/apt/lists/* && \ - for f in /usr/share/openmpi/mpicc-wrapper-data.txt /usr/share/openmpi/mpicc.openmpi-wrapper-data.txt; do \ - if [ -f "${f}" ]; then sed -i "s|^compiler=.*$|compiler=${CC}|" "${f}" ; fi ; \ - done && \ - for f in /usr/share/openmpi/mpic++-wrapper-data.txt /usr/share/openmpi/mpic++.openmpi-wrapper-data.txt /usr/share/openmpi/mpicxx-wrapper-data.txt /usr/share/openmpi/mpicxx.openmpi-wrapper-data.txt /usr/share/openmpi/mpiCC-wrapper-data.txt /usr/share/openmpi/mpiCC.openmpi-wrapper-data.txt; do \ - if [ -f "${f}" ]; then sed -i "s|^compiler=.*$|compiler=${CXX}|" "${f}" ; fi ; \ - done && \ - mpicc --showme:command && \ - mpic++ --showme:command ; \ + apt-get clean && rm -rf /var/lib/apt/lists/* ; \ + fi + +# Install exactly one source-built MPI provider, then expose it through a +# stable path shared by the rest of this Dockerfile and downstream GEOS jobs. +RUN --mount=src=.,dst=$SRC_DIR,ro \ + if [ "${HBV4_BUILD}" = 1 ]; then \ + export HBV4_COMPILER_FLAGS='-march=native -mtune=native' \ + CFLAGS='-march=native -mtune=native' \ + CXXFLAGS='-march=native -mtune=native' \ + FCFLAGS='-march=native -mtune=native' \ + FFLAGS='-march=native -mtune=native' ; \ + fi && \ + if [ "${MPI_PROVIDER}" = openmpi ]; then \ + bash "${SRC_DIR}/docker/install-openmpi.sh" && \ + ln -s /opt/openmpi/5.0.10 /opt/mpi ; \ + else \ + bash "${SRC_DIR}/docker/install-mpich.sh" && \ + ln -s /opt/mpich/5.0.1 /opt/mpi ; \ fi +# Common one-provider runtime contract. CUDA base images' existing library +# search path is retained when present. +ARG LD_LIBRARY_PATH +ENV MPI_PROVIDER=${MPI_PROVIDER} \ + MPI_HOME=${MPI_PREFIX} \ + MPICC=${MPI_PREFIX}/bin/mpicc \ + MPICXX=${MPI_PREFIX}/bin/mpicxx \ + MPIFC=${MPI_PREFIX}/bin/mpifort \ + MPIEXEC=${MPIEXEC_PATH} \ + PATH=${MPI_PREFIX}/bin:${PATH} \ + LD_LIBRARY_PATH=${MPI_PREFIX}/lib:${MPI_PREFIX}/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + +# Build mpi4py from its verified source archive against the selected wrapper. +RUN --mount=src=.,dst=$SRC_DIR,ro \ + if [ "${HBV4_BUILD}" = 1 ]; then \ + export CFLAGS='-march=native -mtune=native' \ + CXXFLAGS='-march=native -mtune=native' \ + FCFLAGS='-march=native -mtune=native' \ + FFLAGS='-march=native -mtune=native' ; \ + fi && \ + bash "${SRC_DIR}/docker/install-mpi4py.sh" + # ----- TPL build stage ----- FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain ARG SRC_DIR ARG BLD_DIR ARG SPEC +ARG DOCKER_BASE_IMAGE +ARG MPI_PROVIDER +ARG MPI_PREFIX +ARG MPIEXEC_PATH +ARG SPACK_ENV_FILE +ARG HBV4_BUILD +ARG SOURCE_SHA RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ @@ -126,29 +191,85 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ echo "ERROR: SPEC build-arg must be supplied" >&2 ; \ exit 1 ; \ fi && \ - GEOSX_SPACK_ENV_FILE=${SRC_DIR}/docker/ubuntu-spack.yaml && \ + case "${SPACK_ENV_FILE}" in \ + docker/ubuntu-spack.yaml|\ + docker/ubuntu-hbv4-openmpi-spack.yaml|\ + docker/ubuntu-hbv4-mpich-spack.yaml) ;; \ + *) echo "ERROR: unrecognized SPACK_ENV_FILE" >&2; exit 1 ;; \ + esac && \ + GEOSX_SPACK_ENV_FILE=${SRC_DIR}/${SPACK_ENV_FILE} && \ + if [ "${HBV4_BUILD}" = 1 ]; then \ + export SPACK_DISABLE_LOCAL_CONFIG=true && \ + unset SCCACHE SCCACHE_DIR RUSTC_WRAPPER ; \ + fi && \ if echo "${CC:-}" | grep -q "clang"; then \ GEOSX_SPACK_ENV_FILE=/tmp/geosx-spack.yaml && \ - cp ${SRC_DIR}/docker/ubuntu-spack.yaml ${GEOSX_SPACK_ENV_FILE} && \ + cp ${SRC_DIR}/${SPACK_ENV_FILE} ${GEOSX_SPACK_ENV_FILE} && \ sed -i -E "s/gcc@([0-9]+) languages:='c,c\\+\\+,fortran'/gcc@\\1 languages:='fortran'/g" ${GEOSX_SPACK_ENV_FILE} && \ sed -i -E '/c: \/usr\/bin\/gcc-[0-9]+/d; /cxx: \/usr\/bin\/g\+\+-[0-9]+/d' ${GEOSX_SPACK_ENV_FILE} ; \ fi && \ - ./scripts/uberenv/uberenv.py \ - --spec "${GEOSX_SPEC}" \ - --spack-env-file=${GEOSX_SPACK_ENV_FILE} \ - --project-json=${SRC_DIR}/.uberenv_config.json \ - --prefix ${GEOSX_TPL_DIR} \ - -k && \ + if [ "${HBV4_BUILD}" = 1 ]; then \ + ./scripts/uberenv/uberenv.py \ + --spec "${GEOSX_SPEC}" \ + --spack-env-file=${GEOSX_SPACK_ENV_FILE} \ + --project-json=${SRC_DIR}/.uberenv_config.json \ + --prefix ${GEOSX_TPL_DIR} \ + --setup-and-env-only \ + -k && \ + bash "${SRC_DIR}/scripts/hbv4/install-spack-from-source.sh" \ + "${GEOSX_TPL_DIR}/spack/bin/spack" \ + "${GEOSX_TPL_DIR}/spack_env" \ + lvarray_hostconfig ; \ + else \ + ./scripts/uberenv/uberenv.py \ + --spec "${GEOSX_SPEC}" \ + --spack-env-file=${GEOSX_SPACK_ENV_FILE} \ + --project-json=${SRC_DIR}/.uberenv_config.json \ + --prefix ${GEOSX_TPL_DIR} \ + -k ; \ + fi && \ rm -f lvarray* && \ cp *.cmake /spack-generated.cmake && \ cd ${GEOSX_TPL_DIR} && \ + mkdir -p /opt/GEOS/hbv4-build-evidence && \ + if [ "${HBV4_BUILD}" = 1 ]; then \ + if [ "${MPI_PROVIDER}" = openmpi ]; then MPI_VERSION=5.0.10; else MPI_VERSION=5.0.1; fi && \ + H5PCC_LIST="$(find "${GEOSX_TPL_DIR}" -path '*/bin/h5pcc' -executable -print)" && \ + [ "$(printf '%s\n' "${H5PCC_LIST}" | sed '/^$/d' | wc -l | tr -d ' ')" = 1 ] || \ + { echo "ERROR: expected exactly one executable h5pcc in the TPL install" >&2; exit 1; } && \ + SPACK_LOCK=${GEOSX_TPL_DIR}/spack_env/spack.lock && \ + [ -r "${SPACK_LOCK}" ] || \ + { echo "ERROR: generated Spack lockfile is missing" >&2; exit 1; } && \ + { find "${GEOSX_TPL_DIR}" -type f \ + \( -name spack-build-env.txt -o -name spack-build-out.txt \) \ + -exec grep -H -E -- '(-march=native|-mtune=native|CFLAGS|CXXFLAGS|FCFLAGS|FFLAGS)' {} + \ + > /tmp/hbv4-spack-build.log || true; } && \ + grep -q -- '-march=native' /tmp/hbv4-spack-build.log || \ + { echo "ERROR: native compiler flags are absent from Spack build evidence" >&2; exit 1; } && \ + HBV4_TPL_SHA="${SOURCE_SHA}" \ + HBV4_COMPILER_FLAGS='target=zen4 -march=native -mtune=native' \ + HBV4_MPI_WRAPPER="${MPI_PREFIX}/bin/mpicc" \ + HBV4_MPI_PREFIX="${MPI_PREFIX}" \ + HBV4_MPI_VERSION="${MPI_VERSION}" \ + HBV4_MPIEXEC="${MPIEXEC_PATH}" \ + HBV4_H5PCC="${H5PCC_LIST}" \ + HBV4_SPACK_BUILD_LOG=/tmp/hbv4-spack-build.log \ + bash "${SRC_DIR}/scripts/hbv4/collect-build-evidence.sh" \ + /opt/GEOS/hbv4-build-evidence \ + "${MPI_PROVIDER}" \ + "${SOURCE_SHA}" \ + "${DOCKER_BASE_IMAGE}" \ + /opt/mpi-configure-args.txt \ + "${SPACK_LOCK}" ; \ + fi && \ rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ -# ----- Final GEOS-build image ----- -FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain +# ----- Common final GEOS-build image ----- +FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain_common ARG SRC_DIR COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR COPY --from=tpl_toolchain /spack-generated.cmake / +COPY --from=tpl_toolchain /opt/GEOS/hbv4-build-evidence /opt/GEOS/hbv4-build-evidence RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ @@ -158,12 +279,24 @@ RUN apt-get update && \ graphviz \ libxml2-utils \ ninja-build \ - python3-mpi4py \ python3-scipy \ python3-matplotlib \ python3-pytest && \ apt-get clean && rm -rf /var/lib/apt/lists/* -# Install sccache to speed up downstream GEOS builds -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh +# Portable images retain the existing downstream compilation cache contract. +FROM geosx_toolchain_common AS geosx_toolchain_portable +ARG SRC_DIR +RUN --mount=src=.,dst=$SRC_DIR,ro $SRC_DIR/docker/install-sccache.sh ENV SCCACHE=/opt/sccache/bin/sccache + +# Native HBv4 images deliberately carry neither sccache nor SCCACHE so the +# generated-code evidence always reflects a fresh native compilation. +FROM geosx_toolchain_common AS geosx_toolchain_hbv4 +COPY --from=tpl_toolchain /opt/GEOS/hbv4-build-evidence/mpi-provider.json /opt/GEOS/mpi-provider.json +COPY --chmod=0755 scripts/hbv4/validate-hbv4-tpls /opt/GEOS/bin/validate-hbv4-tpls +COPY --chmod=0644 scripts/hbv4/parallel_hdf5_shared.c /opt/GEOS/bin/parallel_hdf5_shared.c + +# Keep the historical final-stage name while selecting a closed variant in the +# validated build tuple above. +FROM geosx_toolchain_${FINAL_IMAGE_VARIANT} AS geosx_toolchain diff --git a/docker/ubuntu-hbv4-mpich-spack.yaml b/docker/ubuntu-hbv4-mpich-spack.yaml new file mode 100644 index 00000000..f186c070 --- /dev/null +++ b/docker/ubuntu-hbv4-mpich-spack.yaml @@ -0,0 +1,250 @@ +spack: + config: + install_tree: + root: $spack/.. + projections: + all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + misc_cache: $spack/../misc_cache + test_stage: $spack/../test_stage + build_stage:: + - $spack/../build_stage + + # Regular TPLs do not need views + view: false + + concretizer: + unify: true + reuse: false + + # Override every lower-precedence user/site/default scope. HBv4 images use + # Spack's checksum-addressed public mirror for source archives only; binary + # reuse remains disabled here and by the install command's --no-cache flag. + mirrors:: + spack-public: + url: https://mirror.spack.io + source: true + binary: false + upstreams:: {} + + # Include shared variants and versions + include: + - ../defaults.yaml + - ../versions.yaml + + # --------------------------------------------------------------------------- + # Toolchains + # + # Aligned with the geosx/ubuntu:* base images produced by + # https://github.com/GEOS-DEV/docker_base_images. Each base image installs + # exactly one of these toolchains; the matrix in + # .github/workflows/docker_build_tpls.yml picks the matching '%' + # selector via the SPEC build-arg. + # --------------------------------------------------------------------------- + toolchains: + gcc-13: + - spec: "cflags=='-march=native -mtune=native'" + - spec: "cxxflags=='-march=native -mtune=native'" + - spec: "fflags=='-march=native -mtune=native'" + - spec: '%c=gcc@13' + when: '%c' + - spec: '%cxx=gcc@13' + when: '%cxx' + - spec: '%fortran=gcc@13' + when: '%fortran' + - spec: '%mpich@=5.0.1 %gcc@13' + when: '%mpi' + + packages: + all: + target: [zen4] + + mpi: + require: + - mpich@=5.0.1 + + hdf5: + require: + - '+mpi +shared' + + zlib-api: + require: + - zlib + + blas: + require: + - "netlib-lapack" + lapack: + require: + - "netlib-lapack" + + # ---- Compilers (point at the actual binaries in /usr/bin) ---- + llvm: + buildable: false + externals: + - spec: llvm@19+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang-19 + cxx: /usr/bin/clang++-19 + - spec: llvm@20+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang-20 + cxx: /usr/bin/clang++-20 + - spec: llvm@22+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang-22 + cxx: /usr/bin/clang++-22 + + gcc: + buildable: false + externals: + - spec: gcc@12 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-12 + cxx: /usr/bin/g++-12 + fortran: /usr/bin/gfortran-12 + - spec: gcc@13 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-13 + cxx: /usr/bin/g++-13 + fortran: /usr/bin/gfortran-13 + - spec: gcc@14 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-14 + cxx: /usr/bin/g++-14 + fortran: /usr/bin/gfortran-14 + - spec: gcc@15 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-15 + cxx: /usr/bin/g++-15 + fortran: /usr/bin/gfortran-15 + + + doxygen: + buildable: false + externals: + - spec: doxygen@1.8.20 + prefix: /usr + + autoconf: + version: [2.71] + buildable: false + externals: + - spec: autoconf@2.71 + prefix: /usr + automake: + version: [1.16.5] + buildable: false + externals: + - spec: automake@1.16.5 + prefix: /usr + libtool: + version: [2.4.7] + buildable: false + externals: + - spec: libtool@2.4.7 + prefix: /usr + cmake: + version: [3.31.9] + buildable: false + externals: + - spec: cmake@3.31.9 + prefix: /usr/local + cuda: + buildable: False + externals: + - spec: cuda@12.9.1 +allow-unsupported-compilers + prefix: /usr/local/cuda + - spec: cuda@13.2.1 +allow-unsupported-compilers + prefix: /usr/local/cuda + + findutils: + version: [4.9.0] + buildable: false + externals: + - spec: findutils@4.9.0 + prefix: /usr + m4: + buildable: false + externals: + - spec: m4@1.4.18 + prefix: /usr + mpfr: + buildable: false + externals: + - spec: mpfr@4.2.0 + prefix: /usr + netlib-lapack: + buildable: false + externals: + - spec: netlib-lapack@3.12.0 + prefix: /usr + netlib-blas: + buildable: false + externals: + - spec: netlib-blas@3.12.0 + prefix: /usr + # ---- MPICH: exact source-built HBv4 provider + mpich: + buildable: false + externals: + - spec: mpich@=5.0.1 %gcc@13 + prefix: /opt/mpich/5.0.1 + perl: + buildable: false + externals: + - spec: perl@5.38.2 + prefix: /usr + pkg-config: + buildable: false + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + py-sphinx: + buildable: false + externals: + - spec: py-sphinx@7.4.7 + prefix: /usr + python: + buildable: false + externals: + - spec: python@3.12 + prefix: /usr + readline: + buildable: false + externals: + - spec: readline@8.2 + prefix: /usr + tar: + buildable: false + externals: + - spec: tar@1.35 + prefix: /usr + unzip: + buildable: false + externals: + - spec: unzip@6.0 + prefix: /usr + xz: + buildable: false + externals: + - spec: xz@5.6.1 + prefix: /usr + zlib: + buildable: false + externals: + - spec: zlib@1.3 + prefix: /usr diff --git a/docker/ubuntu-hbv4-openmpi-spack.yaml b/docker/ubuntu-hbv4-openmpi-spack.yaml new file mode 100644 index 00000000..ace071c9 --- /dev/null +++ b/docker/ubuntu-hbv4-openmpi-spack.yaml @@ -0,0 +1,250 @@ +spack: + config: + install_tree: + root: $spack/.. + projections: + all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + misc_cache: $spack/../misc_cache + test_stage: $spack/../test_stage + build_stage:: + - $spack/../build_stage + + # Regular TPLs do not need views + view: false + + concretizer: + unify: true + reuse: false + + # Override every lower-precedence user/site/default scope. HBv4 images use + # Spack's checksum-addressed public mirror for source archives only; binary + # reuse remains disabled here and by the install command's --no-cache flag. + mirrors:: + spack-public: + url: https://mirror.spack.io + source: true + binary: false + upstreams:: {} + + # Include shared variants and versions + include: + - ../defaults.yaml + - ../versions.yaml + + # --------------------------------------------------------------------------- + # Toolchains + # + # Aligned with the geosx/ubuntu:* base images produced by + # https://github.com/GEOS-DEV/docker_base_images. Each base image installs + # exactly one of these toolchains; the matrix in + # .github/workflows/docker_build_tpls.yml picks the matching '%' + # selector via the SPEC build-arg. + # --------------------------------------------------------------------------- + toolchains: + gcc-13: + - spec: "cflags=='-march=native -mtune=native'" + - spec: "cxxflags=='-march=native -mtune=native'" + - spec: "fflags=='-march=native -mtune=native'" + - spec: '%c=gcc@13' + when: '%c' + - spec: '%cxx=gcc@13' + when: '%cxx' + - spec: '%fortran=gcc@13' + when: '%fortran' + - spec: '%openmpi@=5.0.10 %gcc@13' + when: '%mpi' + + packages: + all: + target: [zen4] + + mpi: + require: + - openmpi@=5.0.10 + + hdf5: + require: + - '+mpi +shared' + + zlib-api: + require: + - zlib + + blas: + require: + - "netlib-lapack" + lapack: + require: + - "netlib-lapack" + + # ---- Compilers (point at the actual binaries in /usr/bin) ---- + llvm: + buildable: false + externals: + - spec: llvm@19+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang-19 + cxx: /usr/bin/clang++-19 + - spec: llvm@20+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang-20 + cxx: /usr/bin/clang++-20 + - spec: llvm@22+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang-22 + cxx: /usr/bin/clang++-22 + + gcc: + buildable: false + externals: + - spec: gcc@12 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-12 + cxx: /usr/bin/g++-12 + fortran: /usr/bin/gfortran-12 + - spec: gcc@13 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-13 + cxx: /usr/bin/g++-13 + fortran: /usr/bin/gfortran-13 + - spec: gcc@14 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-14 + cxx: /usr/bin/g++-14 + fortran: /usr/bin/gfortran-14 + - spec: gcc@15 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-15 + cxx: /usr/bin/g++-15 + fortran: /usr/bin/gfortran-15 + + + doxygen: + buildable: false + externals: + - spec: doxygen@1.8.20 + prefix: /usr + + autoconf: + version: [2.71] + buildable: false + externals: + - spec: autoconf@2.71 + prefix: /usr + automake: + version: [1.16.5] + buildable: false + externals: + - spec: automake@1.16.5 + prefix: /usr + libtool: + version: [2.4.7] + buildable: false + externals: + - spec: libtool@2.4.7 + prefix: /usr + cmake: + version: [3.31.9] + buildable: false + externals: + - spec: cmake@3.31.9 + prefix: /usr/local + cuda: + buildable: False + externals: + - spec: cuda@12.9.1 +allow-unsupported-compilers + prefix: /usr/local/cuda + - spec: cuda@13.2.1 +allow-unsupported-compilers + prefix: /usr/local/cuda + + findutils: + version: [4.9.0] + buildable: false + externals: + - spec: findutils@4.9.0 + prefix: /usr + m4: + buildable: false + externals: + - spec: m4@1.4.18 + prefix: /usr + mpfr: + buildable: false + externals: + - spec: mpfr@4.2.0 + prefix: /usr + netlib-lapack: + buildable: false + externals: + - spec: netlib-lapack@3.12.0 + prefix: /usr + netlib-blas: + buildable: false + externals: + - spec: netlib-blas@3.12.0 + prefix: /usr + # ---- OpenMPI: exact source-built HBv4 provider + openmpi: + buildable: false + externals: + - spec: openmpi@=5.0.10 %gcc@13 + prefix: /opt/openmpi/5.0.10 + perl: + buildable: false + externals: + - spec: perl@5.38.2 + prefix: /usr + pkg-config: + buildable: false + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + py-sphinx: + buildable: false + externals: + - spec: py-sphinx@7.4.7 + prefix: /usr + python: + buildable: false + externals: + - spec: python@3.12 + prefix: /usr + readline: + buildable: false + externals: + - spec: readline@8.2 + prefix: /usr + tar: + buildable: false + externals: + - spec: tar@1.35 + prefix: /usr + unzip: + buildable: false + externals: + - spec: unzip@6.0 + prefix: /usr + xz: + buildable: false + externals: + - spec: xz@5.6.1 + prefix: /usr + zlib: + buildable: false + externals: + - spec: zlib@1.3 + prefix: /usr diff --git a/docker/ubuntu-spack.yaml b/docker/ubuntu-spack.yaml index e27514bf..c12a2e78 100644 --- a/docker/ubuntu-spack.yaml +++ b/docker/ubuntu-spack.yaml @@ -109,7 +109,7 @@ spack: mpi: require: - - openmpi@4.1.6 + - openmpi@=5.0.10 zlib-api: require: @@ -242,24 +242,24 @@ spack: externals: - spec: netlib-blas@3.12.0 prefix: /usr - # ---- OpenMPI: same prefix for every compiler binding (Ubuntu 24.04 ships one) + # ---- OpenMPI: source-built in the image with the selected base compiler openmpi: buildable: false externals: - - spec: openmpi@4.1.6 %clang@19 - prefix: /usr - - spec: openmpi@4.1.6 %clang@20 - prefix: /usr - - spec: openmpi@4.1.6 %clang@22 - prefix: /usr - - spec: openmpi@4.1.6 %gcc@12 - prefix: /usr - - spec: openmpi@4.1.6 %gcc@13 - prefix: /usr - - spec: openmpi@4.1.6 %gcc@14 - prefix: /usr - - spec: openmpi@4.1.6 %gcc@15 - prefix: /usr + - spec: openmpi@=5.0.10 %clang@19 + prefix: /opt/openmpi/5.0.10 + - spec: openmpi@=5.0.10 %clang@20 + prefix: /opt/openmpi/5.0.10 + - spec: openmpi@=5.0.10 %clang@22 + prefix: /opt/openmpi/5.0.10 + - spec: openmpi@=5.0.10 %gcc@12 + prefix: /opt/openmpi/5.0.10 + - spec: openmpi@=5.0.10 %gcc@13 + prefix: /opt/openmpi/5.0.10 + - spec: openmpi@=5.0.10 %gcc@14 + prefix: /opt/openmpi/5.0.10 + - spec: openmpi@=5.0.10 %gcc@15 + prefix: /opt/openmpi/5.0.10 perl: buildable: false externals: diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 59218646..738b620c 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -1,5 +1,17 @@ #!/bin/bash -env +# Workflows have historically invoked this script with `bash -x`; disable xtrace +# before handling build metadata so unrelated environment values are not leaked. +set +x +set -eo pipefail + +: "${DOCKER_REPOSITORY:?DOCKER_REPOSITORY must be set}" +: "${DOCKER_TAG:?DOCKER_TAG must be set}" +: "${INSTALL_DIR_ROOT:?INSTALL_DIR_ROOT must be set}" +: "${COMMIT:?COMMIT must be set}" +: "${DOCKER_BASE_IMAGE:?DOCKER_BASE_IMAGE must be set}" +: "${HOST_CONFIG:?HOST_CONFIG must be set}" +: "${SPEC:?SPEC must be set}" +: "${TPL_DOCKERFILE:?TPL_DOCKERFILE must be set}" # We save memory for the docker context echo .git > .dockerignore @@ -28,8 +40,63 @@ echo "Docker base image is ${DOCKER_BASE_IMAGE}" # Optional build-args are only forwarded when set, so the Dockerfiles can rely # on `[ -z "${ARG}" ]` checks. EXTRA_BUILD_ARGS=() -if [ -n "${GCC_VERSION}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "GCC_VERSION=${GCC_VERSION}"); fi -if [ -n "${CLANG_VERSION}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "CLANG_VERSION=${CLANG_VERSION}"); fi +if [ -n "${GCC_VERSION:-}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "GCC_VERSION=${GCC_VERSION}"); fi +if [ -n "${CLANG_VERSION:-}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "CLANG_VERSION=${CLANG_VERSION}"); fi + +# Ubuntu MPI selections are a closed tuple. Prefixes and launchers are derived, +# never accepted from the caller, so image metadata cannot disagree with the +# provider that was compiled into the image. +HBV4_DOCKER_ARGS=() +if [[ "${TPL_DOCKERFILE}" == *tpl-ubuntu.Dockerfile ]]; then + MPI_PROVIDER=${MPI_PROVIDER:-openmpi} + SPACK_ENV_FILE=${SPACK_ENV_FILE:-docker/ubuntu-spack.yaml} + HBV4_BUILD=${HBV4_BUILD:-0} + SOURCE_SHA=${SOURCE_SHA:-} + + case "${MPI_PROVIDER}:${SPACK_ENV_FILE}:${HBV4_BUILD}" in + openmpi:docker/ubuntu-spack.yaml:0) + MPI_PREFIX=/opt/openmpi/5.0.10 + MPIEXEC_PATH=${MPI_PREFIX}/bin/mpirun + FINAL_IMAGE_VARIANT=portable + ;; + openmpi:docker/ubuntu-hbv4-openmpi-spack.yaml:1) + MPI_PREFIX=/opt/openmpi/5.0.10 + MPIEXEC_PATH=${MPI_PREFIX}/bin/mpirun + FINAL_IMAGE_VARIANT=hbv4 + ;; + mpich:docker/ubuntu-hbv4-mpich-spack.yaml:1) + MPI_PREFIX=/opt/mpich/5.0.1 + MPIEXEC_PATH=${MPI_PREFIX}/bin/mpiexec + FINAL_IMAGE_VARIANT=hbv4 + ;; + *) + echo "ERROR: invalid MPI_PROVIDER/SPACK_ENV_FILE/HBV4_BUILD selection" >&2 + exit 1 + ;; + esac + + if [ "${HBV4_BUILD}" = 1 ] && ! [[ "${SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "ERROR: SOURCE_SHA must be exactly 40 lowercase hexadecimal characters for HBv4 builds" >&2 + exit 1 + fi + if [ "${HBV4_BUILD}" = 1 ] && ! [[ "${DOCKER_BASE_IMAGE}" =~ ^.+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: HBv4 DOCKER_BASE_IMAGE must be pinned by sha256 digest" >&2 + exit 1 + fi + + EXTRA_BUILD_ARGS+=( + --build-arg "MPI_PROVIDER=${MPI_PROVIDER}" + --build-arg "MPI_PREFIX=${MPI_PREFIX}" + --build-arg "MPIEXEC_PATH=${MPIEXEC_PATH}" + --build-arg "SPACK_ENV_FILE=${SPACK_ENV_FILE}" + --build-arg "HBV4_BUILD=${HBV4_BUILD}" + --build-arg "SOURCE_SHA=${SOURCE_SHA}" + --build-arg "FINAL_IMAGE_VARIANT=${FINAL_IMAGE_VARIANT}" + ) + if [ "${HBV4_BUILD}" = 1 ]; then + HBV4_DOCKER_ARGS=(--pull --no-cache) + fi +fi BUILDER_ARGS=() if [ -n "${DOCKER_BUILDER:-}" ]; then BUILDER_ARGS+=(--builder "${DOCKER_BUILDER}"); fi @@ -37,16 +104,17 @@ if [ "${DOCKER_LOAD:-0}" = 1 ]; then BUILDER_ARGS+=(--load); fi docker build --progress=plain \ "${BUILDER_ARGS[@]}" \ - --build-arg HOST_CONFIG=${HOST_CONFIG} \ - --build-arg DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE} \ - --build-arg INSTALL_DIR=${INSTALL_DIR} \ + "${HBV4_DOCKER_ARGS[@]}" \ + --build-arg "HOST_CONFIG=${HOST_CONFIG}" \ + --build-arg "DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE}" \ + --build-arg "INSTALL_DIR=${INSTALL_DIR}" \ --build-arg SPEC="${SPEC}" \ "${EXTRA_BUILD_ARGS[@]}" \ - --tag ${DOCKER_REPOSITORY}:${DOCKER_TAG} \ - --file ${TPL_DOCKERFILE} \ + --tag "${DOCKER_REPOSITORY}:${DOCKER_TAG}" \ + --file "${TPL_DOCKERFILE}" \ --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \ --label "org.opencontainers.image.source=https://github.com/GEOS-DEV/thirdPartyLibs" \ - --label "org.opencontainers.image.revision=${COMMIT}" \ + --label "org.opencontainers.image.revision=${SOURCE_SHA:-${COMMIT}}" \ --label "org.opencontainers.image.base.name=${DOCKER_BASE_IMAGE}" \ --label "org.opencontainers.image.title=Building environment for GEOS" \ . diff --git a/scripts/hbv4/README.md b/scripts/hbv4/README.md new file mode 100644 index 00000000..eff8cc86 --- /dev/null +++ b/scripts/hbv4/README.md @@ -0,0 +1,66 @@ +# HBv4 TPL validation assets + +These scripts split tester-host facts from fresh-container image validation. +They make no cloud API calls and never dump the process environment. + +Run `validate-host.sh` on the tester VM before launching a container: + +```sh +scripts/hbv4/validate-host.sh \ + --instance-type Standard_HB176rs_v4 \ + --compiler-flags 'target=zen4 -march=native -mtune=native -O3' \ + --writable-path /mnt/hbv4-local/container-tmp +``` + +The instance type must come from the CI/cloud control-plane metadata step. The +script independently checks the 9V33X CPU identity, 176 visible CPUs, AVX-512, +explicit Zen 4 target evidence, required `-march=native` and `-mtune=native`, +and the exact two-member md RAID0 XFS mount at `/mnt/hbv4-local`. It rejects +`-mcpu=native`. The required writable path must resolve to that same XFS RAID +mount. The mount root intentionally remains root-owned; callers should select a +runner-owned workspace or the mode-`1777` `container-tmp` child. + +Embed `validate-hbv4-tpls` and `parallel_hdf5_shared.c` together at +`/opt/GEOS/bin/`. In a fresh tester container, bind-mount the validated host +mount at (for example) `/hbv4-scratch`, then run: + +```sh +/opt/GEOS/bin/validate-hbv4-tpls --provider openmpi --scratch /hbv4-scratch +/opt/GEOS/bin/validate-hbv4-tpls --provider mpich --scratch /hbv4-scratch +``` + +These commands belong in their respective images, not in one combined image. +The OpenMPI image runs the shared-file test once with `io/ompio` and once with +`io/romio341`; the MPICH image runs it with ROMIO. All MPI tests use four ranks. +Before changing the test environment, the validator checks the image-provided +`MPI_PROVIDER`, `MPI_HOME`, wrapper/launcher variables, `PATH`, and loader path. +It requires all four MPI executables to resolve below the exact provider prefix, +requires the alternate provider tree to be absent, and rejects Debian MPI +packages plus MPI libraries or commands under `/usr`. The package database check +is skipped only on images without `dpkg-query`; filesystem checks still run. +The fresh-container validator first verifies every checksum and field under +`/opt/GEOS/hbv4-build-evidence`. The image integration must also copy or link +the checksummed `mpi-provider.json` to `/opt/GEOS/mpi-provider.json`; the two +files must match byte-for-byte. A sanitized Spack build log is mandatory for a +passing image, even though its collector input variable is optional to support +non-HBv4 diagnostic use of the collector. + +Build evidence is collected with this stable interface: + +```sh +HBV4_TPL_SHA="$tpl_sha" HBV4_COMPILER_FLAGS="$compiler_flags" \ + scripts/hbv4/collect-build-evidence.sh \ + /opt/GEOS/hbv4-build-evidence openmpi "$source_sha" "$base_digest" \ + /tmp/mpi-configure-args.txt /tmp/generated-spack-concretization.txt +``` + +`SOURCE_SHA` and `HBV4_TPL_SHA` are lowercase, 40-hex repository commits; +`BASE_IMAGE` must be an immutable `repository@sha256:<64 lowercase hex>` base +image reference. The provider's pinned MPI +archive SHA-256 is recorded separately. The final positional input must be an +actual generated Spack lock/spec or concretization log, not merely the source +manifest. Set `HBV4_SPACK_BUILD_LOG` to copy one sanitized Spack build log. +`HBV4_MPIEXEC` can explicitly name the provider launcher; it must resolve below +the exact provider prefix. Other optional path/version variables are documented +by `--help`; they are read only when explicitly supplied. The output is a +`manifest.json`, referenced evidence files, and `checksums.sha256`. diff --git a/scripts/hbv4/capture-mpi-wrapper-show.sh b/scripts/hbv4/capture-mpi-wrapper-show.sh new file mode 100755 index 00000000..8818307e --- /dev/null +++ b/scripts/hbv4/capture-mpi-wrapper-show.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +export LC_ALL=C + +die() { printf 'capture-mpi-wrapper-show: ERROR: %s\n' "$*" >&2; exit 1; } + +[[ $# -eq 3 ]] || die 'usage: capture-mpi-wrapper-show.sh PROVIDER MPI_WRAPPER OUTPUT_FILE' + +provider=$1 +mpi_wrapper=$2 +output_file=$3 + +[[ -x "$mpi_wrapper" ]] || die "MPI wrapper is not executable: $mpi_wrapper" +[[ -n "$output_file" ]] || die 'output file must not be empty' + +# Open MPI's mpicc is a symlink to opal_wrapper. The generic wrapper selects +# its configuration from argv[0], so execute the reviewed mpicc path rather +# than its readlink-resolved target. MPICH uses its provider-specific -show +# interface; Open MPI documents --showme for the equivalent full command. +case "$provider" in + openmpi) + "$mpi_wrapper" --showme >"$output_file" 2>&1 || + die 'Open MPI wrapper --showme command failed' + ;; + mpich) + "$mpi_wrapper" -show >"$output_file" 2>&1 || + die 'MPICH wrapper -show command failed' + ;; + *) die 'PROVIDER must be openmpi or mpich' ;; +esac + +[[ -s "$output_file" ]] || die 'MPI wrapper show command produced no evidence' diff --git a/scripts/hbv4/collect-build-evidence.sh b/scripts/hbv4/collect-build-evidence.sh new file mode 100755 index 00000000..bbd185e2 --- /dev/null +++ b/scripts/hbv4/collect-build-evidence.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +export LC_ALL=C + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) +parallel_hdf5_pattern='Parallel HDF5:[[:space:]]*(yes|on)([[:space:]]|$)' + +die() { printf 'collect-build-evidence: ERROR: %s\n' "$*" >&2; exit 1; } +usage() { + cat >&2 <<'EOF' +usage: collect-build-evidence.sh OUTPUT_DIR PROVIDER SOURCE_SHA BASE_IMAGE MPI_CONFIGURE_ARGS_FILE SPACK_CONCRETIZATION_FILE + +Required explicit variables: HBV4_TPL_SHA, HBV4_COMPILER_FLAGS +Optional explicit variables: HBV4_MPI_PREFIX, HBV4_MPI_VERSION, +HBV4_MPI_WRAPPER, HBV4_MPIEXEC, HBV4_H5PCC, HBV4_SPACK_BUILD_LOG + +SPACK_CONCRETIZATION_FILE must be generated lock/spec evidence, not the input +manifest used to request concretization. +EOF +} + +[[ $# -eq 6 ]] || { usage; exit 2; } +output_dir=$1 +provider=$2 +source_sha=$3 +base_image=$4 +configure_args_file=$5 +spack_concretization_file=$6 +: "${HBV4_TPL_SHA:?HBV4_TPL_SHA must be supplied explicitly}" +: "${HBV4_COMPILER_FLAGS:?HBV4_COMPILER_FLAGS must be supplied explicitly}" + +case "$provider" in + openmpi) + default_prefix=/opt/openmpi/5.0.10 + default_version=5.0.10 + default_launcher=mpirun + mpi_source_sha256=0acecc4fc218e5debdbcb8a41d182c6b0f1d29393015ed763b2a91d5d7374cc6 + ;; + mpich) + default_prefix=/opt/mpich/5.0.1 + default_version=5.0.1 + default_launcher=mpiexec + mpi_source_sha256=8c1832a13ddacf071685069f5fadfd1f2877a29e1a628652892c65211b1f3327 + ;; + *) die 'PROVIDER must be openmpi or mpich' ;; +esac +mpi_prefix_requested=${HBV4_MPI_PREFIX:-$default_prefix} +mpi_prefix=$(readlink -f "$mpi_prefix_requested") || die "cannot resolve MPI prefix: $mpi_prefix_requested" +[[ "$mpi_prefix" == "$default_prefix" ]] || die "MPI prefix must resolve to $default_prefix, found $mpi_prefix" +mpi_version=${HBV4_MPI_VERSION:-$default_version} +mpi_wrapper_requested=${HBV4_MPI_WRAPPER:-$mpi_prefix/bin/mpicc} +mpiexec_requested=${HBV4_MPIEXEC:-$mpi_prefix/bin/$default_launcher} +h5pcc=${HBV4_H5PCC:-$(command -v h5pcc || true)} + +[[ "$source_sha" =~ ^[0-9a-f]{40}$ ]] || die 'SOURCE_SHA must be a lowercase 40-hex commit' +[[ "$HBV4_TPL_SHA" =~ ^[0-9a-f]{40}$ ]] || die 'HBV4_TPL_SHA must be a lowercase 40-hex commit' +for scalar_name in base_image mpi_prefix mpi_version mpiexec_requested; do + scalar=${!scalar_name} + [[ "$scalar" =~ ^[A-Za-z0-9_./:@+-]+$ ]] || die "$scalar_name contains unsupported characters" +done +[[ "$HBV4_COMPILER_FLAGS" != *$'\n'* ]] || die 'HBV4_COMPILER_FLAGS must be one line' +[[ -f "$configure_args_file" && -r "$configure_args_file" ]] || die 'MPI configure args file is unreadable' +[[ -f "$spack_concretization_file" && -r "$spack_concretization_file" ]] || die 'Spack concretization file is unreadable' +[[ -x "$mpi_wrapper_requested" ]] || die "MPI wrapper is not executable: $mpi_wrapper_requested" +[[ -x "$mpiexec_requested" ]] || die "MPI launcher is not executable: $mpiexec_requested" +[[ "${mpiexec_requested##*/}" == "$default_launcher" ]] || die "requested MPI launcher must be $default_launcher for $provider" +mpi_wrapper_resolved=$(readlink -f "$mpi_wrapper_requested") || die 'cannot resolve MPI wrapper' +mpiexec=$(readlink -f "$mpiexec_requested") || die 'cannot resolve MPI launcher' +[[ "$mpi_wrapper_resolved" == "$mpi_prefix"/* ]] || die 'MPI wrapper resolves outside exact provider prefix' +[[ "$mpiexec" == "$mpi_prefix"/* ]] || die 'MPI launcher resolves outside exact provider prefix' +[[ -n "$h5pcc" && -x "$h5pcc" ]] || die "h5pcc is not executable: $h5pcc" +command -v lscpu >/dev/null || die 'lscpu is required for evidence capture' +command -v sha256sum >/dev/null || die 'sha256sum is required for evidence capture' +command -v python3 >/dev/null || die 'python3 is required for JSON evidence generation' +[[ "$base_image" =~ ^[A-Za-z0-9._:/+-]+@sha256:[0-9a-f]{64}$ ]] || + die 'BASE_IMAGE must be an immutable repository@sha256:<64 lowercase hex> reference' + +mkdir -p -- "$output_dir" +[[ -d "$output_dir" && -w "$output_dir" ]] || die "output directory is not writable: $output_dir" +[[ -z "$(find "$output_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]] || + die 'output directory must be empty to prevent stale or unreviewed evidence' + +assert_sanitized() { + local evidence_file=$1 + if grep -Eiq '(password|passwd|secret|token|api[_-]?key|authorization|cookie)[[:space:]]*[:=]|https?://[^/@[:space:]]+:[^/@[:space:]]+@' "$evidence_file"; then + die "refusing evidence file with credential-like content: $evidence_file" + fi +} +assert_sanitized "$configure_args_file" +assert_sanitized "$spack_concretization_file" +install -m 0444 "$configure_args_file" "$output_dir/mpi-configure-args.txt" +install -m 0444 "$spack_concretization_file" "$output_dir/spack-concretization.txt" +if [[ -n "${HBV4_SPACK_BUILD_LOG:-}" ]]; then + [[ -f "$HBV4_SPACK_BUILD_LOG" && -r "$HBV4_SPACK_BUILD_LOG" ]] || die 'HBV4_SPACK_BUILD_LOG is unreadable' + assert_sanitized "$HBV4_SPACK_BUILD_LOG" + install -m 0444 "$HBV4_SPACK_BUILD_LOG" "$output_dir/spack-build.log" +fi +printf '%s\n' "$HBV4_COMPILER_FLAGS" >"$output_dir/compiler-flags.txt" +printf '%s\n' "$base_image" >"$output_dir/base-image.txt" +lscpu >"$output_dir/lscpu.txt" +bash "$script_dir/capture-mpi-wrapper-show.sh" \ + "$provider" "$mpi_wrapper_requested" "$output_dir/mpi-wrapper-show.txt" || + die 'MPI wrapper show command failed' +"$h5pcc" -showconfig >"$output_dir/hdf5-config.txt" 2>&1 || die 'h5pcc -showconfig failed' +grep -Eiq "$parallel_hdf5_pattern" "$output_dir/hdf5-config.txt" || + die 'h5pcc reports non-parallel HDF5' + +python3 - "$output_dir/mpi-provider.json" "$provider" "$mpi_version" "$mpi_prefix" \ + "$configure_args_file" "$mpi_source_sha256" "$HBV4_TPL_SHA" "$base_image" \ + "$HBV4_COMPILER_FLAGS" <<'PY' +import json +import sys + +(output, provider, version, prefix, configure_path, archive_sha256, + tpl_source_sha, base_digest, compiler_flags) = sys.argv[1:] +with open(configure_path, encoding="utf-8") as stream: + configure_arguments = stream.read() +record = { + "schema_version": 1, + "provider": provider, + "version": version, + "exact_prefix": prefix, + "configure_arguments": configure_arguments, + "archive_sha256": archive_sha256, + "tpl_source_sha": tpl_source_sha, + "base_digest": base_digest, + "compiler_flags": compiler_flags, +} +with open(output, "w", encoding="utf-8") as stream: + json.dump(record, stream, indent=2, sort_keys=True) + stream.write("\n") +PY + +cat >"$output_dir/manifest.json" <"$checksum_tmp" +) +install -m 0444 "$checksum_tmp" "$output_dir/checksums.sha256" +chmod 0444 "$output_dir"/* +printf 'build evidence written to %s\n' "$output_dir" >&2 diff --git a/scripts/hbv4/install-spack-from-source.sh b/scripts/hbv4/install-spack-from-source.sh new file mode 100755 index 00000000..8a8b5014 --- /dev/null +++ b/scripts/hbv4/install-spack-from-source.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +export LC_ALL=C + +die() { printf 'install-spack-from-source: ERROR: %s\n' "$*" >&2; exit 1; } + +[[ $# -eq 3 ]] || + die 'usage: install-spack-from-source.sh SPACK_EXECUTABLE SPACK_ENVIRONMENT FINAL_PHASE' + +spack_executable=$1 +spack_environment=$2 +final_phase=$3 +fetch_attempts=${HBV4_FETCH_ATTEMPTS:-5} +retry_delay_seconds=${HBV4_FETCH_RETRY_DELAY_SECONDS:-10} +readonly concurrent_packages=4 +readonly global_build_jobs=176 + +[[ -x "$spack_executable" ]] || die "Spack executable is not executable: $spack_executable" +[[ -d "$spack_environment" && -r "$spack_environment/spack.yaml" ]] || + die "Spack environment is missing or unreadable: $spack_environment" +[[ "$final_phase" =~ ^[A-Za-z0-9_+-]+$ ]] || die 'final phase contains unsupported characters' +if ! [[ "$fetch_attempts" =~ ^[1-9][0-9]*$ ]] || (( fetch_attempts > 10 )); then + die 'HBV4_FETCH_ATTEMPTS must be between 1 and 10' +fi +if ! [[ "$retry_delay_seconds" =~ ^[0-9]+$ ]] || (( retry_delay_seconds > 300 )); then + die 'HBV4_FETCH_RETRY_DELAY_SECONDS must be between 0 and 300' +fi + +spack() { + "$spack_executable" -D "$spack_environment" "$@" +} + +# Concretize before starting any paid compilation, then fetch the complete DAG. +# Spack keeps successfully verified archives in its source cache, so a retry +# downloads only sources that did not complete on an earlier attempt. +spack concretize --fresh +spack spec --fresh --install-status --very-long + +fetch_succeeded=false +for (( attempt = 1; attempt <= fetch_attempts; ++attempt )); do + printf 'HBv4 source prefetch attempt %d/%d\n' "$attempt" "$fetch_attempts" >&2 + if spack -k fetch --dependencies; then + fetch_succeeded=true + break + fi + + if (( attempt < fetch_attempts )); then + delay=$((retry_delay_seconds * attempt)) + printf 'HBv4 source prefetch failed; retrying in %d seconds\n' "$delay" >&2 + sleep "$delay" + fi +done + +[[ "$fetch_succeeded" == true ]] || + die "source prefetch failed after $fetch_attempts attempts" + +# The HBv4 manifests configure a source-only mirror, but every concrete node +# must still compile locally. --no-cache makes that policy explicit and blocks +# binary buildcache reuse independently of the mirror configuration. The pinned +# Spack new installer shares one 176-token jobserver across at most four active +# package builds, matching the 176 CPUs exposed to the HBv4 guest. +spack -k install --fresh --keep-stage --no-cache \ + -p "$concurrent_packages" -j "$global_build_jobs" -u "$final_phase" diff --git a/scripts/hbv4/parallel_hdf5_shared.c b/scripts/hbv4/parallel_hdf5_shared.c new file mode 100644 index 00000000..a88bb1a4 --- /dev/null +++ b/scripts/hbv4/parallel_hdf5_shared.c @@ -0,0 +1,129 @@ +#include +#include + +#include +#include +#include + +static int any_rank_failed(int local_failure) +{ + int global_failure = 0; + if (MPI_Allreduce(&local_failure, &global_failure, 1, MPI_INT, MPI_MAX, + MPI_COMM_WORLD) != MPI_SUCCESS) { + return 1; + } + return global_failure; +} + +int main(int argc, char **argv) +{ + hid_t file_access = H5I_INVALID_HID; + hid_t file = H5I_INVALID_HID; + hid_t file_space = H5I_INVALID_HID; + hid_t memory_space = H5I_INVALID_HID; + hid_t dataset = H5I_INVALID_HID; + hid_t transfer = H5I_INVALID_HID; + int mpi_initialized = 0; + int rank = -1; + int size = 0; + int local_failure = 0; + int global_failure = 0; + int value = 0; + int observed = -1; + hsize_t dimensions[1]; + hsize_t start[1]; + hsize_t count[1] = {1}; + const char *path = NULL; + + if (MPI_Init(&argc, &argv) != MPI_SUCCESS) { + fputs("MPI_Init failed\n", stderr); + return EXIT_FAILURE; + } + mpi_initialized = 1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + if (argc != 2 || size != 4) { + if (rank == 0) { + fprintf(stderr, "usage: %s FILE (must run with exactly four ranks)\n", + argv[0]); + } + local_failure = 1; + goto cleanup; + } + path = argv[1]; + dimensions[0] = (hsize_t)size; + start[0] = (hsize_t)rank; + value = 0x48420000 + rank; + + file_access = H5Pcreate(H5P_FILE_ACCESS); + local_failure = file_access < 0 || + H5Pset_fapl_mpio(file_access, MPI_COMM_WORLD, + MPI_INFO_NULL) < 0; + if (any_rank_failed(local_failure)) goto cleanup; + + file = H5Fcreate(path, H5F_ACC_TRUNC, H5P_DEFAULT, file_access); + local_failure = file < 0; + if (any_rank_failed(local_failure)) goto cleanup; + + file_space = H5Screate_simple(1, dimensions, NULL); + dataset = file_space < 0 + ? H5I_INVALID_HID + : H5Dcreate2(file, "rank-values", H5T_NATIVE_INT, file_space, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + local_failure = file_space < 0 || dataset < 0; + if (any_rank_failed(local_failure)) goto cleanup; + + if (H5Sclose(file_space) < 0) local_failure = 1; + file_space = H5I_INVALID_HID; + file_space = H5Dget_space(dataset); + memory_space = H5Screate_simple(1, count, NULL); + transfer = H5Pcreate(H5P_DATASET_XFER); + local_failure = local_failure || file_space < 0 || memory_space < 0 || + transfer < 0; + if (!local_failure) { + local_failure = H5Sselect_hyperslab(file_space, H5S_SELECT_SET, start, NULL, + count, NULL) < 0 || + H5Pset_dxpl_mpio(transfer, H5FD_MPIO_COLLECTIVE) < 0; + } + if (any_rank_failed(local_failure)) goto cleanup; + + local_failure = H5Dwrite(dataset, H5T_NATIVE_INT, memory_space, file_space, + transfer, &value) < 0; + if (any_rank_failed(local_failure)) goto cleanup; + local_failure = H5Fflush(file, H5F_SCOPE_GLOBAL) < 0; + if (any_rank_failed(local_failure)) goto cleanup; + + local_failure = H5Dread(dataset, H5T_NATIVE_INT, memory_space, file_space, + transfer, &observed) < 0 || + observed != value; + global_failure = any_rank_failed(local_failure); + if (global_failure && rank == 0) { + fputs("collective HDF5 read-back verification failed\n", stderr); + } + +cleanup: + global_failure = any_rank_failed(local_failure || global_failure); + if (transfer >= 0 && H5Pclose(transfer) < 0) global_failure = 1; + if (memory_space >= 0 && H5Sclose(memory_space) < 0) global_failure = 1; + if (file_space >= 0 && H5Sclose(file_space) < 0) global_failure = 1; + if (dataset >= 0 && H5Dclose(dataset) < 0) global_failure = 1; + if (file >= 0 && H5Fclose(file) < 0) global_failure = 1; + if (file_access >= 0 && H5Pclose(file_access) < 0) global_failure = 1; + + if (mpi_initialized) { + int close_failure = any_rank_failed(global_failure); + MPI_Barrier(MPI_COMM_WORLD); + if (rank == 0 && path != NULL && unlink(path) != 0) { + perror("unlink shared HDF5 test file"); + close_failure = 1; + } + global_failure = any_rank_failed(close_failure); + MPI_Finalize(); + } + + if (!global_failure && rank == 0) { + puts("parallel HDF5 shared-file verification passed with four ranks"); + } + return global_failure ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/scripts/hbv4/validate-hbv4-tpls b/scripts/hbv4/validate-hbv4-tpls new file mode 100755 index 00000000..17726593 --- /dev/null +++ b/scripts/hbv4/validate-hbv4-tpls @@ -0,0 +1,430 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +export LC_ALL=C +parallel_hdf5_pattern='Parallel HDF5:[[:space:]]*(yes|on)([[:space:]]|$)' + +die() { printf 'validate-hbv4-tpls: ERROR: %s\n' "$*" >&2; exit 1; } +note() { printf 'validate-hbv4-tpls: %s\n' "$*" >&2; } + +usage() { + cat >&2 <<'EOF' +usage: validate-hbv4-tpls --provider openmpi|mpich --scratch DIR + +The image must embed this file at /opt/GEOS/bin/validate-hbv4-tpls and embed +parallel_hdf5_shared.c beside it. DIR must be a writable bind-mounted scratch +directory supplied by the fresh tester container. +EOF +} + +provider= +scratch= +while (($#)); do + case "$1" in + --provider) (($# >= 2)) || die '--provider needs a value'; provider=$2; shift 2 ;; + --scratch) (($# >= 2)) || die '--scratch needs a value'; scratch=$2; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done + +case "$provider" in + openmpi) + mpi_prefix=/opt/openmpi/5.0.10 + mpi_version=5.0.10 + launcher=mpirun + other_prefix=/opt/mpich + ;; + mpich) + mpi_prefix=/opt/mpich/5.0.1 + mpi_version=5.0.1 + launcher=mpiexec + other_prefix=/opt/openmpi + ;; + *) die '--provider must be openmpi or mpich' ;; +esac + +[[ -n "$scratch" && -d "$scratch" && -w "$scratch" ]] || + die '--scratch must name an existing writable directory' + +# Validate the environment baked into the image before normalizing any MPI +# variables for the test itself. /opt/mpi symlinks are accepted only when their +# canonical targets are below the exact versioned provider prefix. +[[ "${MPI_PROVIDER-}" == "$provider" ]] || + die "image MPI_PROVIDER (${MPI_PROVIDER-unset}) does not match $provider" +[[ -n "${MPI_HOME-}" && -d "$MPI_HOME" ]] || die 'image MPI_HOME is missing or invalid' +mpi_home_real=$(readlink -f "$MPI_HOME") || die "cannot resolve image MPI_HOME: $MPI_HOME" +[[ "$mpi_home_real" == "$mpi_prefix" ]] || + die "image MPI_HOME resolves to $mpi_home_real, expected $mpi_prefix" + +validate_image_executable() { + local variable=$1 expected_name=$2 value resolved + value=${!variable-} + [[ -n "$value" && "$value" == /* && -x "$value" ]] || + die "image $variable is not an absolute executable path: ${value:-unset}" + [[ "${value##*/}" == "$expected_name" ]] || + die "image $variable must name $expected_name, found ${value##*/}" + resolved=$(readlink -f "$value") || die "cannot resolve image $variable: $value" + [[ "$resolved" == "$mpi_prefix"/* ]] || + die "image $variable resolves outside $mpi_prefix: $resolved" +} +validate_image_executable MPICC mpicc +validate_image_executable MPICXX mpicxx +validate_image_executable MPIFC mpifort +validate_image_executable MPIEXEC "$launcher" + +for required_command in mpicc mpicxx mpifort "$launcher"; do + command_path=$(command -v "$required_command") || die "$required_command is absent from image PATH" + command_real=$(readlink -f "$command_path") || die "cannot resolve PATH command $command_path" + [[ "$command_real" == "$mpi_prefix"/* ]] || + die "image PATH selects $required_command outside $mpi_prefix: $command_real" +done + +[[ -n "${LD_LIBRARY_PATH-}" ]] || die 'image LD_LIBRARY_PATH is unset or empty' +loader_has_provider=0 +IFS=: read -r -a loader_entries <<<"$LD_LIBRARY_PATH" +for loader_entry in "${loader_entries[@]}"; do + [[ -n "$loader_entry" ]] || continue + loader_real=$(readlink -m "$loader_entry") || die "cannot normalize loader entry: $loader_entry" + [[ "$loader_real" != "$other_prefix" && "$loader_real" != "$other_prefix"/* ]] || + die "image loader path contains the other MPI provider: $loader_entry" + if [[ "$loader_real" == "$mpi_prefix"/* ]]; then + if [[ -d "$loader_entry" ]]; then + loader_has_provider=1 + fi + fi +done +[[ "$loader_has_provider" -eq 1 ]] || die "image loader path has no directory below $mpi_prefix" + +[[ ! -e "$other_prefix" ]] || die "other MPI provider prefix exists: $other_prefix" +if command -v dpkg-query >/dev/null 2>&1; then + installed_mpi_packages=$( + dpkg-query -W -f='${binary:Package}\t${Status}\n' 2>/dev/null | + awk -F '\t' '$2 == "install ok installed" { + package = $1; sub(/:.*/, "", package) + if (package ~ /(openmpi|mpich|mpi-default|mpi4py)/) print package + }' + ) || die 'failed to query installed Debian packages' + [[ -z "$installed_mpi_packages" ]] || + die "Debian MPI packages are installed: ${installed_mpi_packages//$'\n'/, }" +else + note 'dpkg-query unavailable; skipping Debian package database check' +fi + +usr_mpi_artifact=$(find /usr \( -type f -o -type l \) \ + \( -name 'libmpi*.so*' -o -name 'libmpi*.a' \) -print -quit 2>/dev/null) || + die 'failed to inspect /usr for MPI libraries' +[[ -z "$usr_mpi_artifact" ]] || die "MPI library artifact exists under /usr: $usr_mpi_artifact" +for usr_mpi_command in /usr/bin/mpicc /usr/bin/mpicxx /usr/bin/mpifort \ + /usr/bin/mpirun /usr/bin/mpiexec; do + [[ ! -e "$usr_mpi_command" ]] || die "MPI command artifact exists under /usr: $usr_mpi_command" +done + +export MPI_PROVIDER=$provider +export MPI_HOME=$mpi_prefix +export MPIEXEC="$mpi_prefix/bin/$launcher" +export MPICC="$mpi_prefix/bin/mpicc" +export MPICXX="$mpi_prefix/bin/mpicxx" +export MPIFC="$mpi_prefix/bin/mpifort" +export PATH="$mpi_prefix/bin:$PATH" +if [[ -d "$mpi_prefix/lib64" ]]; then + export LD_LIBRARY_PATH="$mpi_prefix/lib:$mpi_prefix/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +else + export LD_LIBRARY_PATH="$mpi_prefix/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +fi + +under_prefix() { + local candidate resolved + candidate=$(command -v "$1") || die "$1 is not in PATH" + resolved=$(readlink -f "$candidate") || die "cannot resolve $candidate" + [[ "$resolved" == "$mpi_prefix"/* ]] || + die "$1 resolves outside selected MPI prefix: $resolved" +} +under_prefix "$launcher" +under_prefix mpicc +under_prefix mpicxx +under_prefix mpifort +[[ "$MPI_HOME" == "$mpi_prefix" ]] || die 'MPI_HOME does not select requested MPI' + +case ":$PATH:" in + *":$other_prefix/"*) die "PATH contains the other MPI provider: $other_prefix" ;; +esac +case ":${LD_LIBRARY_PATH:-}:" in + *":$other_prefix/"*) die "LD_LIBRARY_PATH contains the other MPI provider: $other_prefix" ;; +esac + +if [[ "$provider" == openmpi ]]; then + under_prefix ompi_info + ompi_info --version | grep -F "$mpi_version" >/dev/null || die 'OpenMPI version mismatch' + ompi_components=$(ompi_info --parsable --param io all) + grep -Eq '(^|:)mca:io:ompio:' <<<"$ompi_components" || die 'OpenMPI io/ompio is missing' + grep -Eq '(^|:)mca:io:romio341:' <<<"$ompi_components" || die 'OpenMPI io/romio341 is missing' +else + under_prefix mpichversion + mpich_details=$(mpichversion) + grep -Eq "MPICH Version:[[:space:]]*$mpi_version([[:space:]]|$)" <<<"$mpich_details" || + die 'MPICH version mismatch' + "$mpi_prefix/bin/mpiexec" -info 2>&1 | grep -qi hydra || die 'Hydra launcher evidence is missing' + grep -Eqi 'romio|--enable-romio' <<<"$mpich_details" || die 'ROMIO evidence is missing from mpichversion' +fi + +tpl_root=${GEOSX_TPL_DIR:-/opt/GEOS} +[[ -d "$tpl_root" ]] || die "TPL root does not exist: $tpl_root" +h5pcc=$(command -v h5pcc || true) +if [[ -z "$h5pcc" ]]; then + mapfile -t h5pcc_candidates < <(find "$tpl_root" -type f -name h5pcc -perm -111 -print) + [[ ${#h5pcc_candidates[@]} -eq 1 ]] || + die "expected exactly one executable h5pcc below $tpl_root, found ${#h5pcc_candidates[@]}" + h5pcc=${h5pcc_candidates[0]} +fi +h5pcc=$(readlink -f "$h5pcc") || die "cannot resolve h5pcc: $h5pcc" +[[ "$h5pcc" == "$tpl_root"/* ]] || die "h5pcc resolves outside the TPL root: $h5pcc" +hdf5_config=$($h5pcc -showconfig) +grep -Eiq "$parallel_hdf5_pattern" <<<"$hdf5_config" || die 'h5pcc reports non-parallel HDF5' + +# h5pcc supplies -L/-lhdf5 but does not add the HDF5 installation directory +# to the executable's runtime search path. Expose only that verified prefix for +# the dynamic test; the exact selected MPI prefix remains independently checked. +hdf5_prefix=$(CDPATH='' cd -- "$(dirname -- "$h5pcc")/.." && pwd -P) || + die "cannot resolve the HDF5 prefix from $h5pcc" +hdf5_loader_entries=() +for candidate in "$hdf5_prefix/lib" "$hdf5_prefix/lib64"; do + [[ -d "$candidate" ]] || continue + compgen -G "$candidate/libhdf5.so*" >/dev/null || continue + hdf5_loader_entries+=("$candidate") +done +[[ ${#hdf5_loader_entries[@]} -gt 0 ]] || + die "HDF5 shared library is missing below $hdf5_prefix" +hdf5_loader_path=$(IFS=:; printf '%s' "${hdf5_loader_entries[*]}") +export LD_LIBRARY_PATH="$hdf5_loader_path${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + +python_bin=$(command -v python3 || command -v python) || die 'Python is missing' +evidence_dir=/opt/GEOS/hbv4-build-evidence +[[ -d "$evidence_dir" ]] || die "embedded build evidence is missing: $evidence_dir" +[[ -f "$evidence_dir/checksums.sha256" ]] || die 'embedded evidence checksums are missing' +( + cd -- "$evidence_dir" + sha256sum --check --strict checksums.sha256 +) || die 'embedded build evidence checksum verification failed' + +"$python_bin" - "$evidence_dir" "$provider" "$mpi_version" "$mpi_prefix" <<'PY' +import json +import os +import re +import sys + +directory, provider, version, prefix = sys.argv[1:] +manifest_path = os.path.join(directory, "manifest.json") +try: + with open(manifest_path, encoding="utf-8") as stream: + manifest = json.load(stream) +except (OSError, ValueError) as error: + raise SystemExit("invalid evidence manifest: %s" % error) + +expected_archives = { + "openmpi": "0acecc4fc218e5debdbcb8a41d182c6b0f1d29393015ed763b2a91d5d7374cc6", + "mpich": "8c1832a13ddacf071685069f5fadfd1f2877a29e1a628652892c65211b1f3327", +} +expected_artifacts = { + "compiler_flags": "compiler-flags.txt", + "lscpu": "lscpu.txt", + "mpi_configure_args": "mpi-configure-args.txt", + "mpi_wrapper": "mpi-wrapper-show.txt", + "hdf5_config": "hdf5-config.txt", + "mpi_provider": "mpi-provider.json", + "spack_concretization": "spack-concretization.txt", + "spack_build_log": "spack-build.log", +} +expected_launcher = {"openmpi": "mpirun", "mpich": "mpiexec"}[provider] +launcher_requested = str(manifest.get("mpi_launcher_requested", "")) +launcher_resolved = str(manifest.get("mpi_launcher_resolved", "")) +checks = ( + (manifest.get("schema_version") == 1, "schema version"), + (manifest.get("provider") == provider, "provider"), + (manifest.get("mpi_version") == version, "MPI version"), + (manifest.get("mpi_prefix") == prefix, "MPI prefix"), + (manifest.get("mpi_source_sha256") == expected_archives[provider], "MPI archive checksum"), + (bool(re.fullmatch(r"[0-9a-f]{40}", str(manifest.get("source_sha", "")))), "source SHA"), + (bool(re.fullmatch(r"[0-9a-f]{40}", str(manifest.get("tpl_source_sha", "")))), "TPL source SHA"), + (manifest.get("source_sha") == manifest.get("tpl_source_sha"), "source/TPL SHA agreement"), + (os.path.basename(launcher_requested) == expected_launcher, "requested MPI launcher"), + (launcher_resolved.startswith(prefix + os.sep), "MPI launcher prefix"), + (os.path.realpath(launcher_requested) == launcher_resolved, "MPI launcher resolution"), +) +for valid, label in checks: + if not valid: + raise SystemExit("evidence manifest mismatch: " + label) + +artifacts = manifest.get("artifacts") +if artifacts != expected_artifacts: + raise SystemExit("evidence manifest has missing or unexpected artifact paths") +for relative in artifacts.values(): + path = os.path.realpath(os.path.join(directory, relative)) + if os.path.commonpath((os.path.realpath(directory), path)) != os.path.realpath(directory): + raise SystemExit("evidence artifact escapes evidence directory: " + relative) + if not os.path.isfile(path): + raise SystemExit("evidence artifact is missing: " + relative) + +with open(os.path.join(directory, "checksums.sha256"), encoding="utf-8") as stream: + listed = {line.split(None, 1)[1].lstrip("*").removeprefix("./").strip() for line in stream if line.strip()} +actual = { + name for name in os.listdir(directory) + if name != "checksums.sha256" and os.path.isfile(os.path.join(directory, name)) +} +if listed != actual: + raise SystemExit("checksum index does not cover exactly the embedded evidence files") + +provider_record_path = os.path.join(directory, artifacts["mpi_provider"]) +try: + with open(provider_record_path, encoding="utf-8") as stream: + provider_record = json.load(stream) + with open(os.path.join(directory, artifacts["mpi_configure_args"]), encoding="utf-8") as stream: + configure_arguments = stream.read() + with open(os.path.join(directory, artifacts["compiler_flags"]), encoding="utf-8") as stream: + compiler_flags = stream.read().removesuffix("\n") + with open(os.path.join(directory, artifacts["spack_concretization"]), encoding="utf-8") as stream: + spack_lock = json.load(stream) +except (OSError, ValueError) as error: + raise SystemExit("invalid MPI provider evidence: %s" % error) +expected_provider_record = { + "schema_version": 1, + "provider": provider, + "version": version, + "exact_prefix": prefix, + "configure_arguments": configure_arguments, + "archive_sha256": manifest["mpi_source_sha256"], + "tpl_source_sha": manifest["tpl_source_sha"], + "base_digest": manifest.get("base_digest"), + "compiler_flags": compiler_flags, +} +if provider_record != expected_provider_record: + raise SystemExit("mpi-provider.json does not exactly match manifest/artifact evidence") +if not re.fullmatch(r"[A-Za-z0-9._:/+\-]+@sha256:[0-9a-f]{64}", str(provider_record["base_digest"])): + raise SystemExit("MPI provider base image is not an immutable digest") +if not isinstance(spack_lock, dict): + raise SystemExit("Spack lockfile root is not an object") +concrete_specs = spack_lock.get("concrete_specs") +if not isinstance(concrete_specs, dict) or not concrete_specs: + raise SystemExit("Spack lockfile does not contain concrete specs") +if not all(isinstance(spec, dict) for spec in concrete_specs.values()): + raise SystemExit("Spack lockfile contains an invalid concrete spec") +source_specs = [spec for spec in concrete_specs.values() if "external" not in spec] +if not source_specs: + raise SystemExit("Spack lockfile does not contain source-built specs") +source_targets = { + spec.get("arch", {}).get("target") + for spec in source_specs +} +if source_targets != {"x86_64_v4"}: + raise SystemExit( + "source-built Spack specs do not all target x86_64_v4: %s" + % sorted(str(target) for target in source_targets) + ) +spec_names = {spec.get("name") for spec in concrete_specs.values()} +if provider not in spec_names: + raise SystemExit("Spack concretization does not contain the selected MPI provider") +other_provider = "mpich" if provider == "openmpi" else "openmpi" +if other_provider in spec_names: + raise SystemExit("Spack concretization contains the other MPI provider") +PY + +provider_record=/opt/GEOS/mpi-provider.json +[[ -f "$provider_record" ]] || die "top-level MPI provider record is missing: $provider_record" +cmp -s "$evidence_dir/mpi-provider.json" "$provider_record" || + die 'top-level MPI provider record differs from checksummed evidence' + +require_native_flags() { + local evidence_file=$1 + grep -F -- '-march=native' "$evidence_file" >/dev/null || + die "$evidence_file lacks -march=native" + grep -F -- '-mtune=native' "$evidence_file" >/dev/null || + die "$evidence_file lacks -mtune=native" + ! grep -F -- '-mcpu=native' "$evidence_file" >/dev/null || + die "$evidence_file contains forbidden -mcpu=native" +} +require_native_flags "$evidence_dir/compiler-flags.txt" +grep -Eq '(^|[[:space:]])(target=zen4|arch=zen4)([[:space:]]|$)' "$evidence_dir/compiler-flags.txt" || + die 'compiler-flags evidence lacks target=zen4' +require_native_flags "$evidence_dir/mpi-configure-args.txt" +require_native_flags "$evidence_dir/mpi-wrapper-show.txt" +require_native_flags "$evidence_dir/spack-build.log" +grep -Eiq '^Model name:.*AMD EPYC.*9V33X' "$evidence_dir/lscpu.txt" || + die 'embedded builder CPU evidence is not AMD EPYC 9V33X' +grep -Eq '^CPU\(s\):[[:space:]]*176[[:space:]]*$' "$evidence_dir/lscpu.txt" || + die 'embedded builder CPU evidence does not show 176 CPUs' +grep -Eiq '^Flags:.*[[:space:]]avx512f([[:space:]]|$)' "$evidence_dir/lscpu.txt" || + die 'embedded builder CPU evidence lacks AVX-512' +grep -Eiq "$parallel_hdf5_pattern" "$evidence_dir/hdf5-config.txt" || + die 'embedded HDF5 evidence is not parallel-enabled' + +script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P) +source_file=$script_dir/parallel_hdf5_shared.c +[[ -r "$source_file" ]] || die "embedded test source is missing: $source_file" +work_dir=$(mktemp -d "$scratch/hbv4-tpl-validation.XXXXXX") || die 'cannot create scratch directory' +cleanup() { rm -rf -- "$work_dir"; } +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +test_exe=$work_dir/parallel-hdf5-shared +"$h5pcc" -O2 -Wall -Wextra -Werror "$source_file" -o "$test_exe" + +assert_selected_libmpi() { + local target=$1 report=$2 library + local -a mpi_libraries=() + if ! ldd "$target" >"$report" 2>&1; then + cat "$report" >&2 + die "ldd failed for $target" + fi + if grep -q 'not found' "$report"; then + cat "$report" >&2 + die "unresolved library in $target" + fi + grep -F "$other_prefix/" "$report" >/dev/null && die "$target links the other MPI provider" + mapfile -t mpi_libraries < <( + awk '/(^|[[:space:]])libmpi[^[:space:]]*/ { + for (i = 1; i <= NF; ++i) if ($i == "=>") print $(i + 1) + }' "$report" + ) + [[ ${#mpi_libraries[@]} -gt 0 ]] || die "$target has no dynamic libmpi dependency" + for library in "${mpi_libraries[@]}"; do + [[ "$library" == /* ]] || die "$target has unresolved libmpi entry: $library" + library=$(readlink -f "$library") || die "cannot resolve linked library $library" + [[ "$library" == "$mpi_prefix"/* ]] || + die "$target resolves libmpi outside $mpi_prefix: $library" + done +} +assert_selected_libmpi "$test_exe" "$work_dir/hdf5-test.ldd" + +mpi4py_module=$($python_bin - <<'PY' +import mpi4py +from mpi4py import MPI +if mpi4py.__version__ != "4.1.2": + raise SystemExit("mpi4py version is %s, expected 4.1.2" % mpi4py.__version__) +print(MPI.__file__) +print(MPI.Get_library_version().replace("\n", " ")) +PY +) || die 'mpi4py import/version/library query failed' +mpi4py_extension=${mpi4py_module%%$'\n'*} +[[ -f "$mpi4py_extension" ]] || die "mpi4py MPI extension is not a file: $mpi4py_extension" +assert_selected_libmpi "$mpi4py_extension" "$work_dir/mpi4py.ldd" +mpi_library_version=${mpi4py_module#*$'\n'} +case "$provider:$mpi_library_version" in + openmpi:*Open\ MPI*v5.0.10*|openmpi:*Open\ MPI*5.0.10*) ;; + mpich:*MPICH*5.0.1*) ;; + *) die "mpi4py reports the wrong MPI library: $mpi_library_version" ;; +esac + +mpi_python='from mpi4py import MPI; assert MPI.COMM_WORLD.Get_size() == 4; print(MPI.COMM_WORLD.Get_rank(), MPI.Get_library_version().splitlines()[0])' +if [[ "$provider" == openmpi ]]; then + export OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + for component in ompio romio341; do + note "running shared parallel HDF5 verification with OpenMPI io/$component" + "$launcher" -n 4 --mca io "$component" "$test_exe" "$work_dir/$component.h5" + done +else + note 'running shared parallel HDF5 verification with MPICH ROMIO' + "$launcher" -n 4 "$test_exe" "$work_dir/romio.h5" +fi +"$launcher" -n 4 "$python_bin" -c "$mpi_python" +note "$provider $mpi_version, parallel HDF5, and mpi4py 4.1.2 validation passed" diff --git a/scripts/hbv4/validate-host.sh b/scripts/hbv4/validate-host.sh new file mode 100755 index 00000000..58e9c98e --- /dev/null +++ b/scripts/hbv4/validate-host.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +export LC_ALL=C + +die() { printf 'validate-host: ERROR: %s\n' "$*" >&2; exit 1; } +note() { printf 'validate-host: %s\n' "$*" >&2; } +usage() { + printf 'usage: %s --instance-type Standard_HB176rs_v4 --compiler-flags STRING --writable-path PATH [--mount PATH]\n' "$0" >&2 +} + +instance_type= +compiler_flags= +mount_path=/mnt/hbv4-local +writable_path= +while (($#)); do + case "$1" in + --instance-type) (($# >= 2)) || die 'missing instance type'; instance_type=$2; shift 2 ;; + --compiler-flags) (($# >= 2)) || die 'missing compiler flags'; compiler_flags=$2; shift 2 ;; + --mount) (($# >= 2)) || die 'missing mount path'; mount_path=$2; shift 2 ;; + --writable-path) (($# >= 2)) || die 'missing writable path'; writable_path=$2; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done + +[[ -n "$writable_path" ]] || die '--writable-path is required' + +[[ "$instance_type" =~ ^Standard_HB[0-9]+[A-Za-z0-9_-]*_v4$ ]] || + die "instance evidence is not an HBv4 SKU: $instance_type" +[[ "$compiler_flags" =~ (^|[[:space:]])(target=zen4|arch=zen4)([[:space:]]|$) ]] || + die 'build evidence does not contain an explicit Zen 4 target' +[[ "$compiler_flags" =~ (^|[[:space:]])-march=native([[:space:]]|$) ]] || + die 'compiler flags do not contain -march=native' +[[ "$compiler_flags" =~ (^|[[:space:]])-mtune=native([[:space:]]|$) ]] || + die 'compiler flags do not contain -mtune=native' +[[ ! "$compiler_flags" =~ (^|[[:space:]])-mcpu=native([[:space:]]|$) ]] || + die '-mcpu=native is forbidden' + +command -v lscpu >/dev/null || die 'lscpu is required' +lscpu_output=$(lscpu) +grep -Eiq '^Model name:.*AMD EPYC.*9V33X' <<<"$lscpu_output" || + die 'CPU model is not AMD EPYC 9V33X HBv4 hardware' +grep -Eiq '^Flags:.*(^|[[:space:]])avx512f([[:space:]]|$)' <<<"$lscpu_output" || + die 'CPU lacks expected Zen 4 AVX-512 evidence' +cpu_count=$(awk -F: '/^CPU\(s\):/{gsub(/[[:space:]]/, "", $2); print $2; exit}' <<<"$lscpu_output") +[[ "$cpu_count" == 176 ]] || die "expected 176 visible HBv4 CPUs, found ${cpu_count:-unknown}" + +command -v findmnt >/dev/null || die 'findmnt is required' +mount_record=$(findmnt -n -T "$mount_path" -o SOURCE,FSTYPE,TARGET) || die "$mount_path is not mounted" +IFS=' ' read -r mount_source mount_type mount_target <<<"$mount_record" +[[ "$mount_target" == "$mount_path" ]] || die "$mount_path is not a distinct mount point" +[[ "$mount_type" == xfs ]] || die "$mount_path must use XFS, found $mount_type" +mount_device=$(readlink -f "$mount_source") || die "cannot resolve mount source $mount_source" +mount_name=${mount_device##*/} +[[ "$mount_name" == md* && -d "/sys/block/$mount_name/md" ]] || + die "$mount_path source is not an md array: $mount_source" +[[ "$(<"/sys/block/$mount_name/md/level")" == raid0 ]] || die 'md array is not RAID0' +[[ "$(<"/sys/block/$mount_name/md/raid_disks")" == 2 ]] || die 'md array does not declare exactly two disks' +shopt -s nullglob +members=("/sys/block/$mount_name/slaves/"*) +shopt -u nullglob +[[ ${#members[@]} -eq 2 ]] || die "md array has ${#members[@]} members, expected exactly two" + +# The XFS mount root intentionally remains root-owned. Prove write access in the +# caller's designated workload directory, and first prove that directory is +# still backed by the exact RAID mount validated above. This catches a lost +# mount without weakening the mount-root permission boundary. +[[ -d "$writable_path" ]] || die "writable path is not a directory: $writable_path" +writable_mount_record=$(findmnt -n -T "$writable_path" -o SOURCE,FSTYPE,TARGET) || + die "cannot resolve writable path mount: $writable_path" +IFS=' ' read -r writable_source writable_type writable_target <<<"$writable_mount_record" +[[ "$writable_target" == "$mount_path" && "$writable_type" == xfs ]] || + die "writable path is not backed by $mount_path XFS: $writable_path" +[[ "$(readlink -f "$writable_source")" == "$mount_device" ]] || + die "writable path is not backed by the validated RAID device: $writable_path" + +probe=$(mktemp "$writable_path/.hbv4-write-test.XXXXXX") || + die "writable path rejected a file probe: $writable_path" +trap 'rm -f -- "$probe"' EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +printf 'hbv4 storage validation\n' >"$probe" +sync "$probe" +note "validated $instance_type: AMD EPYC 9V33X, 176 CPUs, Zen 4 flags, two-disk md RAID0 XFS at $mount_path, writable workload path $writable_path" From b0f718da17e7973d4cbff6acdacd4469d4785e86 Mon Sep 17 00:00:00 2001 From: Randolph Settgast Date: Wed, 12 Aug 2026 22:24:34 -0700 Subject: [PATCH 2/2] Remove synthetic HBv4 target validation Validate the actual Spack target from spack.lock while treating native compiler flags as the build-tuning evidence. Remove handwritten target tokens from host and candidate validation, keep the lock/native/MPI gates, and add regression coverage for the trusted pull_request_target control path. --- .../_docker_build_tpls_hbv4_provider.yml | 4 ++-- ci/azure/tests/test-hbv4-build-resilience.sh | 23 +++++++++++++++---- ci/azure/tests/test-hbv4-workflow-contract.sh | 2 +- scripts/hbv4/README.md | 6 ++--- scripts/hbv4/validate-hbv4-tpls | 2 -- scripts/hbv4/validate-host.sh | 4 +--- 6 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.github/workflows/_docker_build_tpls_hbv4_provider.yml b/.github/workflows/_docker_build_tpls_hbv4_provider.yml index c90827cf..f9b3bc9d 100644 --- a/.github/workflows/_docker_build_tpls_hbv4_provider.yml +++ b/.github/workflows/_docker_build_tpls_hbv4_provider.yml @@ -160,7 +160,7 @@ jobs: run: >- /opt/geos-ci/scripts/hbv4/validate-host.sh --instance-type Standard_HB176rs_v4 - --compiler-flags 'target=zen4 -march=native -mtune=native' + --compiler-flags '-march=native -mtune=native' --writable-path "${GITHUB_WORKSPACE}" - name: Build without cache @@ -539,7 +539,7 @@ jobs: printf 'trusted-parallel-hdf5-source-sha256=%s\n' "${trusted_source_sha256}" /opt/geos-ci/scripts/hbv4/validate-host.sh \ --instance-type Standard_HB176rs_v4 \ - --compiler-flags 'target=zen4 -march=native -mtune=native' \ + --compiler-flags '-march=native -mtune=native' \ --writable-path "${GITHUB_WORKSPACE}" echo 'trusted-host-validation=success' } 2>&1 | tee "${evidence_file}" diff --git a/ci/azure/tests/test-hbv4-build-resilience.sh b/ci/azure/tests/test-hbv4-build-resilience.sh index eef4a85b..8bbec627 100755 --- a/ci/azure/tests/test-hbv4-build-resilience.sh +++ b/ci/azure/tests/test-hbv4-build-resilience.sh @@ -8,6 +8,8 @@ capture_wrapper="${repository_root}/scripts/hbv4/capture-mpi-wrapper-show.sh" install_from_source="${repository_root}/scripts/hbv4/install-spack-from-source.sh" collect_evidence="${repository_root}/scripts/hbv4/collect-build-evidence.sh" container_validator="${repository_root}/scripts/hbv4/validate-hbv4-tpls" +host_validator="${repository_root}/scripts/hbv4/validate-host.sh" +provider_workflow="${repository_root}/.github/workflows/_docker_build_tpls_hbv4_provider.yml" dockerfile="${repository_root}/docker/tpl-ubuntu.Dockerfile" test_root=$(mktemp -d "${TMPDIR:-/tmp}/hbv4-build-resilience.XXXXXX") trap 'rm -rf -- "$test_root"' EXIT @@ -43,15 +45,26 @@ if grep -Eiq "$parallel_hdf5_pattern" <<<'Parallel HDF5: OFF'; then fail 'parallel-HDF5 pattern accepted a disabled value' fi -# Spack normalizes the Zen 4 preference to the x86-64-v4 architecture name in -# its lockfile. Require that target for every source-built spec while allowing -# generic x86_64 targets on compilers and other declared externals. +# Keep the real target check on the generated Spack lockfile. A handwritten +# target token in compiler-flags evidence is not an independent architecture +# signal and must never gate builder or candidate validation. assert_contains "$container_validator" 'source_specs = [spec for spec in concrete_specs.values() if "external" not in spec]' assert_contains "$container_validator" 'if source_targets != {"x86_64_v4"}:' assert_contains "$container_validator" 'source-built Spack specs do not all target x86_64_v4' -if grep -Fq -- 'Spack concretization does not contain target=zen4' "$container_validator"; then - fail 'validator still requires the non-concrete target spelling zen4' +assert_contains "$host_validator" 'compiler flags do not contain -march=native' +assert_contains "$host_validator" 'compiler flags do not contain -mtune=native' +assert_contains "$host_validator" '-mcpu=native is forbidden' +assert_contains "$container_validator" 'require_native_flags "$evidence_dir/compiler-flags.txt"' +assert_contains "$container_validator" 'require_native_flags "$evidence_dir/mpi-configure-args.txt"' +assert_contains "$container_validator" 'require_native_flags "$evidence_dir/mpi-wrapper-show.txt"' +assert_contains "$container_validator" 'require_native_flags "$evidence_dir/spack-build.log"' +if grep -Eq -- '(^|[^[:alnum:]_])(target|arch)=[[:alnum:]_.+-]+' \ + "$host_validator" "$container_validator"; then + fail 'validator still treats a handwritten target token as build proof' fi +[[ $(grep -Fc -- "--compiler-flags '-march=native -mtune=native'" \ + "$provider_workflow") -eq 2 ]] || + fail 'trusted workflow must pass only native tuning flags to both host checks' # h5pcc links shared HDF5 without embedding its installation path. The runtime # check must add only the verified HDF5 prefix and retain strict MPI resolution. diff --git a/ci/azure/tests/test-hbv4-workflow-contract.sh b/ci/azure/tests/test-hbv4-workflow-contract.sh index c00049cf..2ab6b932 100755 --- a/ci/azure/tests/test-hbv4-workflow-contract.sh +++ b/ci/azure/tests/test-hbv4-workflow-contract.sh @@ -279,7 +279,7 @@ assert!(!tester_validation_script.include?('validation_scratch="/mnt/hbv4-local/ "tester must not create scratch directly under the protected RAID mount root") # The workflow declares its root spec alongside its other build configuration; -# provider manifests own the Zen 4 target and native compiler constraints. +# provider manifests own Spack target selection and native compiler constraints. workflow_env = workflow.fetch("env", {}) assert!(workflow_env["SPEC"] == "~pygeosx ~docs %gcc-13", "HBv4 workflow must declare the root spec with terminal %gcc-13") diff --git a/scripts/hbv4/README.md b/scripts/hbv4/README.md index eff8cc86..2081f2fc 100644 --- a/scripts/hbv4/README.md +++ b/scripts/hbv4/README.md @@ -8,14 +8,14 @@ Run `validate-host.sh` on the tester VM before launching a container: ```sh scripts/hbv4/validate-host.sh \ --instance-type Standard_HB176rs_v4 \ - --compiler-flags 'target=zen4 -march=native -mtune=native -O3' \ + --compiler-flags '-march=native -mtune=native -O3' \ --writable-path /mnt/hbv4-local/container-tmp ``` The instance type must come from the CI/cloud control-plane metadata step. The script independently checks the 9V33X CPU identity, 176 visible CPUs, AVX-512, -explicit Zen 4 target evidence, required `-march=native` and `-mtune=native`, -and the exact two-member md RAID0 XFS mount at `/mnt/hbv4-local`. It rejects +required `-march=native` and `-mtune=native`, and the exact two-member md RAID0 +XFS mount at `/mnt/hbv4-local`. It rejects `-mcpu=native`. The required writable path must resolve to that same XFS RAID mount. The mount root intentionally remains root-owned; callers should select a runner-owned workspace or the mode-`1777` `container-tmp` child. diff --git a/scripts/hbv4/validate-hbv4-tpls b/scripts/hbv4/validate-hbv4-tpls index 17726593..66e7c14b 100755 --- a/scripts/hbv4/validate-hbv4-tpls +++ b/scripts/hbv4/validate-hbv4-tpls @@ -343,8 +343,6 @@ require_native_flags() { die "$evidence_file contains forbidden -mcpu=native" } require_native_flags "$evidence_dir/compiler-flags.txt" -grep -Eq '(^|[[:space:]])(target=zen4|arch=zen4)([[:space:]]|$)' "$evidence_dir/compiler-flags.txt" || - die 'compiler-flags evidence lacks target=zen4' require_native_flags "$evidence_dir/mpi-configure-args.txt" require_native_flags "$evidence_dir/mpi-wrapper-show.txt" require_native_flags "$evidence_dir/spack-build.log" diff --git a/scripts/hbv4/validate-host.sh b/scripts/hbv4/validate-host.sh index 58e9c98e..f1504278 100755 --- a/scripts/hbv4/validate-host.sh +++ b/scripts/hbv4/validate-host.sh @@ -28,8 +28,6 @@ done [[ "$instance_type" =~ ^Standard_HB[0-9]+[A-Za-z0-9_-]*_v4$ ]] || die "instance evidence is not an HBv4 SKU: $instance_type" -[[ "$compiler_flags" =~ (^|[[:space:]])(target=zen4|arch=zen4)([[:space:]]|$) ]] || - die 'build evidence does not contain an explicit Zen 4 target' [[ "$compiler_flags" =~ (^|[[:space:]])-march=native([[:space:]]|$) ]] || die 'compiler flags do not contain -march=native' [[ "$compiler_flags" =~ (^|[[:space:]])-mtune=native([[:space:]]|$) ]] || @@ -83,4 +81,4 @@ trap 'exit 130' INT trap 'exit 143' TERM printf 'hbv4 storage validation\n' >"$probe" sync "$probe" -note "validated $instance_type: AMD EPYC 9V33X, 176 CPUs, Zen 4 flags, two-disk md RAID0 XFS at $mount_path, writable workload path $writable_path" +note "validated $instance_type: AMD EPYC 9V33X, 176 CPUs, native compiler flags, two-disk md RAID0 XFS at $mount_path, writable workload path $writable_path"