diff --git a/.github/workflows/sim-tests.yml b/.github/workflows/sim-tests.yml new file mode 100644 index 000000000..60c05358f --- /dev/null +++ b/.github/workflows/sim-tests.yml @@ -0,0 +1,192 @@ +--- +name: Simulation tests + +'on': + pull_request: {} + workflow_dispatch: + inputs: + tag: + description: 'O2PDPSuite tag to test against (default: newest daily)' + type: string + required: false + +permissions: {} + +concurrency: + group: sim-tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + sim-tests: + name: Simulation tests against CVMFS + runs-on: [self-hosted, cvmfs] + timeout-minutes: 180 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + # The changed-file logic diffs against the merge base, so the full + # history is needed, not a shallow clone. + fetch-depth: 0 + + - name: Resolve the diff base + id: base + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -eu + git rev-parse --verify "$BASE_SHA^{commit}" >/dev/null || { + echo "::error title=Cannot resolve diff base::pull request base commit $BASE_SHA does not resolve in this checkout" + exit 1 + } + git rev-parse --verify "$HEAD_SHA^{commit}" >/dev/null || { + echo "::error title=Cannot resolve diff head::pull request head commit $HEAD_SHA does not resolve in this checkout" + exit 1 + } + merge_base=$(git merge-base "$BASE_SHA" "$HEAD_SHA") || { + echo "::error title=Cannot compute diff base::git merge-base failed" + exit 1 + } + [ -n "$merge_base" ] || { + echo "::error title=Cannot compute diff base::merge base is empty" + exit 1 + } + echo "sha=$merge_base" >> "$GITHUB_OUTPUT" + + - name: Skip when not relevant or opted into the source build + id: gate + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ steps.base.outputs.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -eu + changed=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA") + # Opting in means *editing* the existing sentinel, so only count it + # when it is Modified. A pull request that Adds it — which is what the + # pull request introducing the sentinel does — would otherwise trip its + # own opt-out and pass without testing anything. + opted=$(git diff --name-only --diff-filter=M "$BASE_SHA" "$HEAD_SHA") + if grep -qx 'test/needs-o2-dev' <<< "$opted" ; then + echo "::notice title=Skipped::this pull request touches test/needs-o2-dev, so it is tested by build/O2DPG/sim/o2dev against O2 dev instead" + echo "skip=true" >> "$GITHUB_OUTPUT" + elif ! grep -qE '^(DATA/|MC/|test/|RelVal/)' <<< "$changed" ; then + echo "::notice title=Skipped::no changed file matches DATA/, MC/, test/ or RelVal/" + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + + - name: Check the CVMFS environment + if: steps.gate.outputs.skip != 'true' + run: | + set -eu + test -d /cvmfs/alice.cern.ch || { + echo "::error title=CVMFS unavailable::/cvmfs/alice.cern.ch is not mounted on this runner" + exit 1 + } + test -x /cvmfs/alice.cern.ch/bin/alienv || { + echo "::error title=CVMFS unavailable::/cvmfs/alice.cern.ch/bin/alienv is missing" + exit 1 + } + + - name: Check the AliEn token + if: steps.gate.outputs.skip != 'true' + env: + JALIEN_TOKEN_CERT: /run/alien-ci/tokencert.pem + JALIEN_TOKEN_KEY: /run/alien-ci/tokenkey.pem + run: | + set -eu + # DPL's CCDB backend refuses to talk to alice-ccdb.cern.ch without a + # token and aborts the whole device, so check up front rather than + # letting it surface as a confusing task crash deep in a workflow. + for f in "$JALIEN_TOKEN_CERT" "$JALIEN_TOKEN_KEY" ; do + test -r "$f" || { + echo "::error title=No AliEn token::$f is missing or unreadable. On the runner host: systemctl start alien-ci-token.service" + exit 1 + } + done + if ! openssl x509 -in "$JALIEN_TOKEN_CERT" -noout -checkend 3600 >/dev/null 2>&1 ; then + echo "::error title=AliEn token expiring::the token expires within the hour. On the runner host: systemctl start alien-ci-token.service" + exit 1 + fi + echo "AliEn token valid until $(openssl x509 -in "$JALIEN_TOKEN_CERT" -noout -enddate | cut -d= -f2)" + + - name: Resolve the O2PDPSuite tag + id: tag + if: steps.gate.outputs.skip != 'true' + env: + REQUESTED_TAG: ${{ inputs.tag }} + PR_BODY: ${{ github.event.pull_request.body }} + run: | + set -eu + # shellcheck source=test/ci/resolve_tag.sh + . test/ci/resolve_tag.sh + moduledir=/cvmfs/alice.cern.ch/el9-x86_64/Modules/modulefiles/O2PDPSuite + requested=$REQUESTED_TAG + if [ -z "$requested" ]; then + # A PR can pin the release with a line "sim-tests-tag: ". + requested=$(printf '%s\n' "$PR_BODY" | + sed -n 's/^[[:space:]]*sim-tests-tag:[[:space:]]*//p' | head -n 1 | + tr -d '[:space:]') + fi + tag=$(resolve_o2pdpsuite_tag "$moduledir" "$requested") || { + echo "::error title=No usable O2PDPSuite release::see the message above" + exit 1 + } + echo "Testing against O2PDPSuite::$tag" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + + - name: Run the O2DPG tests + if: steps.gate.outputs.skip != 'true' + env: + O2PDPSUITE_TAG: ${{ steps.tag.outputs.tag }} + O2DPG_TEST_HASH_BASE: ${{ steps.base.outputs.sha }} + O2DPG_TEST_HASH_HEAD: ${{ github.event.pull_request.head.sha }} + JOBS: 8 + # o2dpg_sim_workflow.py calls JAlien(['whoami']) purely to fill the + # AOD's --created-by field, and only when JALIEN_USER is unset. Setting + # it keeps the tests from needing a GRID credential at all, which + # matters here: anything readable by this account is readable by the + # fork-pull-request code that runs as it. + JALIEN_USER: alien-ci + # DPL's CCDB backend needs a GRID token for alice-ccdb.cern.ch. These + # are *paths* to a short-lived token minted from a service certificate + # by a root-owned timer on the runner host; the certificate itself is + # never readable by this account. Verified that paths work — the same + # variables also accept PEM content, which is what ali-bot passes. + JALIEN_TOKEN_CERT: /run/alien-ci/tokencert.pem + JALIEN_TOKEN_KEY: /run/alien-ci/tokenkey.pem + run: | + set -eu + # Everything after "-c" is joined into one string and re-evaluated + # by the CVMFS alienv via "bash -c \"$*\"". Quoting here is applied + # once then discarded, so it is safe for the runner's workspace path, + # but a path with a space or "$" would break or double-evaluate. + /cvmfs/alice.cern.ch/bin/alienv setenv "O2PDPSuite/$O2PDPSUITE_TAG" -c \ + env O2DPG_ROOT="$PWD" O2DPG_MC_CONFIG_ROOT="$PWD" \ + O2DPG_TEST_REPO_DIR="$PWD" \ + O2DPG_TEST_HASH_BASE="$O2DPG_TEST_HASH_BASE" \ + O2DPG_TEST_HASH_HEAD="$O2DPG_TEST_HASH_HEAD" \ + JOBS="$JOBS" \ + bash test/run_tests.sh + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: o2dpg-test-logs + path: | + o2dpg_tests/**/*.log + o2dpg_tests/**/*serverlog* + o2dpg_tests/**/*workerlog* + o2dpg_tests/**/*mergerlog* + if-no-files-found: ignore + retention-days: 14 + + - name: Prune test artifacts + if: always() + run: find o2dpg_tests -type f ! -name '*.log' ! -name '*serverlog*' ! -name '*workerlog*' ! -name '*mergerlog*' -delete || true diff --git a/.github/workflows/syntax-checks.yml b/.github/workflows/syntax-checks.yml index bd9b5de6e..73b888523 100644 --- a/.github/workflows/syntax-checks.yml +++ b/.github/workflows/syntax-checks.yml @@ -134,6 +134,24 @@ jobs: - name: Run the FileIOGraph test suite run: python3 -m unittest discover -s UTILS/FileIOGraph/tests -t UTILS/FileIOGraph/tests + bash-harnesses: + name: Test-harness unit tests + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run the offline test harnesses + run: | + error=0 + for t in test/tests/*.sh ; do + echo "::group::$t" + bash "$t" || error=1 + echo "::endgroup::" + done + exit "$error" + pylint: name: Pylint runs-on: ubuntu-latest diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 8f48700e6..551c96d87 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -5,6 +5,7 @@ # It aims to handle the different MC possible configurations # It just creates a workflow.json txt file, to execute the workflow one must execute right after # ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json +# The tests covering this script are described in test/README.md. # # Execution examples: # - pp PYTHIA jets, 2 events, triggered on high pT decay photons on all barrel calorimeters acceptance, eCMS 13 TeV diff --git a/test/README.md b/test/README.md index 7109e4e70..276e9eaf7 100644 --- a/test/README.md +++ b/test/README.md @@ -7,7 +7,7 @@ At the moment, the tests focus on generator configurations and custom generators Tests are run via ```bash -${O2DPG_ROOT}/test/run_tests.sh [--fail-immediately] +${O2DPG_ROOT}/test/run_tests.sh [--fail-immediately] [--keep-artifacts] [SUBTEST...] ``` Tests are run for changed @@ -56,7 +56,32 @@ O2DPG_TEST_REPO_DIR= ${O2DPG_ROOT}/test/run_tests.sh [--f ``` If you are inside the source directory, you can simply run ```bash -${O2DPG_ROOT}/test/run_tests.sh [--fail-immediately] +${O2DPG_ROOT}/test/run_tests.sh [--fail-immediately] [--keep-artifacts] [SUBTEST...] +``` +If the change you are testing is to a test script itself (`run_tests.sh` or +any `run_*_tests.sh`), invoke the checkout's own entrypoint instead, e.g. +`bash test/run_tests.sh` from inside the checkout: `run_tests.sh` finds its +sub-scripts next to itself, so calling `${O2DPG_ROOT}/test/run_tests.sh` +tests the *released* copy of the script you just edited, not your change, +even with `O2DPG_TEST_REPO_DIR` pointed at the checkout. + +### Running a subset + +`run_tests.sh` runs the generator, workflow and RelVal sub-tests. To run only +some of them, name them: + +```bash +${O2DPG_ROOT}/test/run_tests.sh generator relval +``` + +The offline harnesses under `test/tests/` check the entrypoint's selection, +exit-code aggregation and O2PDPSuite tag resolution without needing an O2 +environment: + +```bash +bash test/tests/run_tests_selection.sh +bash test/tests/exit_code_aggregation.sh +bash test/tests/resolve_tag.sh ``` ### Keeping all test artifacts @@ -74,24 +99,33 @@ ${O2DPG_ROOT}/test/run_tests.sh -h ``` which will give you ``` -usage: run_tests.sh [--fail-immediately] [--keep-artifacts] + +usage: run_tests.sh [--fail-immediately] [--keep-artifacts] [SUBTEST...] + + SUBTEST : one or more of: generator workflow relval (default: all) FLAGS: - --fail-immediately : abort as soon as the first tests fails - --keep-artifacts : keep simulation and tests artifacts, by default everything but the logs is removed after each test + --fail-immediately : stop after the first failing sub-test + --keep-artifacts : keep simulation artifacts, not just the logs ENVIRONMENT VARIABLES: - O2DPG_TEST_REPO_DIR : Point to the source repository you want to test. - O2DPG_TEST_HASH_BASE : The base hash you want to use for comparison (optional) - O2DPG_TEST_HASH_HEAD : The head hash you want to use for comparison (optional) - - If O2DPG_TEST_HASH_BASE is not set, it will be looked for ALIBUILD_BASE_HASH. - If also not set, this will be set to HEAD~1. However, if there are unstaged - changes, it will be set to HEAD. + O2DPG_TEST_REPO_DIR : the source repository to test + O2DPG_TEST_HASH_BASE : base hash for the changed-file diff (optional) + O2DPG_TEST_HASH_HEAD : head hash for the changed-file diff (optional) - If O2DPG_TEST_HASH_HEAD is not set, it will be looked for ALIBUILD_HEAD_HASH. - If also not set, this will be set to HEAD. However, if there are unstaged - changes, it will left blank. ``` + +## When your change needs an unreleased O2 + +The `Simulation tests against CVMFS` check runs against a published +`O2PDPSuite` release, so a change that depends on an unmerged or unreleased O2 +commit cannot pass it. Two escape hatches, in order of preference: + +1. If the O2 change is already in a published daily, pin it: add a line + `sim-tests-tag: daily-YYYYMMDD-HHMM-1` to the pull request description. +2. If it is not published anywhere yet, touch `test/needs-o2-dev` with a + one-line reason and a link to the O2 pull request. That enables + `build/O2DPG/sim/o2dev`, which builds O2 from source against `dev`. It is + much slower, so it is opt-in. diff --git a/test/ci/resolve_tag.sh b/test/ci/resolve_tag.sh new file mode 100644 index 000000000..940090ba1 --- /dev/null +++ b/test/ci/resolve_tag.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Pick the O2PDPSuite release to test against. Kept free of CVMFS paths and of +# CI variables so it can be exercised offline. + +resolve_o2pdpsuite_tag() +{ + local moduledir=${1:-} + local requested=${2:-} + + if [[ ! -d "${moduledir}" ]] ; then + echo "resolve_o2pdpsuite_tag: no such directory: ${moduledir}" >&2 + return 1 + fi + + if [[ -n "${requested}" ]] ; then + case ${requested} in + *[!A-Za-z0-9._-]* ) + echo "resolve_o2pdpsuite_tag: invalid tag: ${requested}" >&2 + return 1 ;; + esac + if [[ ! -e "${moduledir}/${requested}" ]] ; then + echo "resolve_o2pdpsuite_tag: requested tag not available: ${requested}" >&2 + return 1 + fi + echo "${requested}" + return 0 + fi + + local newest + newest=$(find "${moduledir}" -maxdepth 1 -name 'daily-*' -printf '%f\n' 2>/dev/null | + sort -V | tail -n 1) + if [[ -z "${newest}" ]] ; then + echo "resolve_o2pdpsuite_tag: no daily-* tag in ${moduledir}" >&2 + return 1 + fi + echo "${newest}" +} diff --git a/test/needs-o2-dev b/test/needs-o2-dev new file mode 100644 index 000000000..ee9799078 --- /dev/null +++ b/test/needs-o2-dev @@ -0,0 +1,5 @@ +Touch this file in a pull request that cannot be tested against a published +O2PDPSuite release because it depends on an unreleased change in O2. +Replace this text with a one-line reason and a link to the O2 pull request. +Touching it enables the build/O2DPG/sim/o2dev check, which builds O2 from +source against AliceO2Group/AliceO2 dev. diff --git a/test/run_tests.sh b/test/run_tests.sh new file mode 100755 index 000000000..400f4d5e4 --- /dev/null +++ b/test/run_tests.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Single entrypoint for the O2DPG tests. Runs the generator, workflow and +# RelVal sub-tests, aggregates their exit codes and prints the collected log. + +set -u + +SUBTEST_DIR=${O2DPG_TEST_SUBTEST_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)} + +ALL_SUBTESTS="generator workflow relval" + +print_usage() +{ + echo + echo "usage: run_tests.sh [--fail-immediately] [--keep-artifacts] [SUBTEST...]" + echo + echo " SUBTEST : one or more of: ${ALL_SUBTESTS} (default: all)" + echo + echo " FLAGS:" + echo + echo " --fail-immediately : stop after the first failing sub-test" + echo " --keep-artifacts : keep simulation artifacts, not just the logs" + echo + echo " ENVIRONMENT VARIABLES:" + echo + echo " O2DPG_TEST_REPO_DIR : the source repository to test" + echo " O2DPG_TEST_HASH_BASE : base hash for the changed-file diff (optional)" + echo " O2DPG_TEST_HASH_HEAD : head hash for the changed-file diff (optional)" + echo +} + +fail_immediately= +subtest_args=() +selected=() + +while [ "$#" -gt 0 ] ; do + case $1 in + --fail-immediately ) fail_immediately=1 ; subtest_args+=("$1") ; shift ;; + --keep-artifacts ) subtest_args+=("$1") ; shift ;; + --help|-h ) print_usage ; exit 1 ;; + generator|workflow|relval ) selected+=("$1") ; shift ;; + * ) echo "Unknown argument ${1}" ; print_usage ; exit 1 ;; + esac +done + +[[ "${#selected[@]}" == "0" ]] && read -r -a selected <<< "${ALL_SUBTESTS}" + +# Geant4 dataset variables, if a Geant4 installation is in the environment. +if [[ -n "${G4INSTALL:-}" ]] ; then + eval "$("${G4INSTALL}/bin/geant4-config" --datasets | + sed -e 's/[^ ]* //' -e 's/G4/export G4/' -e 's/DATA /DATA=/')" +fi + +# O2's CCDB dictionary payload includes . An aliBuild build +# environment has curl's headers; a plain runtime environment from CVMFS does +# not, and cling then dies compiling the payload. Derive the prefix from +# curl-config rather than hardcoding a version. +if command -v curl-config > /dev/null 2>&1 ; then + curl_include=$(curl-config --prefix 2>/dev/null)/include + if [[ -f "${curl_include}/curl/curl.h" && ":${ROOT_INCLUDE_PATH:-}:" != *":${curl_include}:"* ]] ; then + export ROOT_INCLUDE_PATH="${curl_include}${ROOT_INCLUDE_PATH:+:${ROOT_INCLUDE_PATH}}" + echo "Added ${curl_include} to ROOT_INCLUDE_PATH for the CCDB dictionary" + fi +fi + +# LHAPDF data, needed by several generator configurations. +if [[ -z "${LHAPDF_DATA_PATH:-}" && -n "${LHAPDF_ROOT:-}" ]] ; then + export LHAPDF_DATA_PATH="${LHAPDF_ROOT}/share/LHAPDF:${LHAPDF_PDFSETS_ROOT:-}/share/LHAPDF" + echo "Set LHAPDF_DATA_PATH to ${LHAPDF_DATA_PATH}" +fi + +ret_global=0 +for subtest in "${selected[@]}" ; do + script="${SUBTEST_DIR}/run_${subtest}_tests.sh" + if [[ ! -x "${script}" ]] ; then + echo "ERROR: sub-test script ${script} not found or not executable" + ret_global=1 + [[ "${fail_immediately}" == "1" ]] && break + continue + fi + echo + echo "==> START SUBTEST: ${subtest} <==" + # Only run_generator_tests.sh accepts --fail-immediately/--keep-artifacts; + # run_workflow_tests.sh and run_relval_tests.sh accept only --help/-h. + case ${subtest} in + generator ) forward=("${subtest_args[@]:-}") ;; + * ) forward=() ;; + esac + "${script}" "${forward[@]}" + ret_this=$? + echo "==> END SUBTEST: ${subtest} (exit ${ret_this}) <==" + if [[ "${ret_this}" != "0" ]] ; then + ret_global=${ret_this} + [[ "${fail_immediately}" == "1" ]] && break + fi +done + +echo +if [[ "${ret_global}" != "0" ]] ; then + echo "error detected in O2DPG tests, see above" +else + echo "O2DPG tests passed" +fi + +exit ${ret_global} diff --git a/test/run_workflow_tests.sh b/test/run_workflow_tests.sh index 5c7fb4117..e6fb3717a 100755 --- a/test/run_workflow_tests.sh +++ b/test/run_workflow_tests.sh @@ -293,7 +293,7 @@ fi # prepare our local test directory for bin tests # global return code for PWGs ret_analysis_qc=0 -if [[ "${changed_analysis_qc}" != "" ]] ; then +if [[ "${changed_analysis_qc}" != "" || "${changed_sim_bin}" != "" ]] ; then rm -rf ${TEST_PARENT_DIR_BIN} 2>/dev/null mkdir -p ${TEST_PARENT_DIR_BIN} 2>/dev/null pushd ${TEST_PARENT_DIR_BIN} > /dev/null @@ -336,7 +336,7 @@ if [[ "${changed_anchored}" != "" ]] ; then popd > /dev/null fi -RET=$(( ret_analysis_qc + ret_global_anchored )) +RET=$(( ret_global_pwg + ret_analysis_qc + ret_global_anchored )) echo [[ "${RET}" != "0" ]] && echo_red "There were errors, please check!" || echo_green "All required workflow tests successful" diff --git a/test/tests/exit_code_aggregation.sh b/test/tests/exit_code_aggregation.sh new file mode 100755 index 000000000..1172f23d9 --- /dev/null +++ b/test/tests/exit_code_aggregation.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Static guard against the exit-code-aggregation bug in run_workflow_tests.sh: +# the final RET=$(( ... )) line must include every aggregate return-code +# variable the script computes, and each of those variables must actually be +# assigned somewhere in the script. + +set -u +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCRIPT=${O2DPG_TEST_WORKFLOW_SCRIPT:-"${HERE}/../run_workflow_tests.sh"} +FAILURES=0 + +check() +{ + local label=$1 expected=$2 actual=$3 + if [[ "${expected}" == "${actual}" ]] ; then + echo "ok - ${label}" + else + echo "FAIL - ${label}: expected '${expected}', got '${actual}'" + FAILURES=$((FAILURES + 1)) + fi +} + +# The three aggregate return-code variables the final exit code must include. +# Explicit and hard-coded on purpose: a fourth aggregate added later must make +# whoever adds it come here and update this list. +REQUIRED_VARS="ret_global_pwg ret_analysis_qc ret_global_anchored" + +RET_LINE=$(grep -m1 -E '^RET=\$\(\(' "${SCRIPT}") + +for var in ${REQUIRED_VARS} ; do + present=$(grep -c -- "${var}" <<< "${RET_LINE}") + check "RET= line includes ${var}" 1 "${present}" +done + +for var in ${REQUIRED_VARS} ; do + assigned=$(grep -c -E "^[[:space:]]*${var}=" "${SCRIPT}") + check "${var} is assigned in the script" 1 "$( [[ "${assigned}" -ge 1 ]] && echo 1 || echo 0 )" +done + +echo +if [[ "${FAILURES}" == "0" ]] ; then + echo "All exit-code aggregation checks passed" +else + echo "${FAILURES} test(s) failed" +fi +exit $(( FAILURES > 0 ? 1 : 0 )) diff --git a/test/tests/resolve_tag.sh b/test/tests/resolve_tag.sh new file mode 100755 index 000000000..d896ece11 --- /dev/null +++ b/test/tests/resolve_tag.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Offline harness for resolve_o2pdpsuite_tag: builds a fake modulefile +# directory so the function can be checked without CVMFS. + +set -u +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../ci/resolve_tag.sh +source "${HERE}/../ci/resolve_tag.sh" +FAILURES=0 + +check() +{ + local label=$1 expected=$2 actual=$3 + if [[ "${expected}" == "${actual}" ]] ; then + echo "ok - ${label}" + else + echo "FAIL - ${label}: expected '${expected}', got '${actual}'" + FAILURES=$((FAILURES + 1)) + fi +} + +TMP=$(mktemp -d) +D="${TMP}/O2PDPSuite" +mkdir -p "${D}" +# Deliberately out of lexical-creation order, and with non-daily neighbours. +touch "${D}/daily-20260919-0000-1" \ + "${D}/daily-20260921-0000-1" \ + "${D}/daily-20260920-0000-1" \ + "${D}/async-2024-pp-apass1-1" \ + "${D}/slc9-nightly-20260921-1" + +out=$(resolve_o2pdpsuite_tag "${D}") ; rc=$? +check "newest daily is chosen" "daily-20260921-0000-1" "${out}" +check "success exit code" 0 "${rc}" + +out=$(resolve_o2pdpsuite_tag "${D}" "daily-20260919-0000-1") ; rc=$? +check "requested tag honoured" "daily-20260919-0000-1" "${out}" +check "requested tag exit code" 0 "${rc}" + +out=$(resolve_o2pdpsuite_tag "${D}" "daily-20991231-0000-1" 2>/dev/null) ; rc=$? +check "absent requested tag fails" 1 "${rc}" +check "absent requested tag prints nothing on stdout" "" "${out}" + +out=$(resolve_o2pdpsuite_tag "${TMP}/does-not-exist" 2>/dev/null) ; rc=$? +check "missing directory fails" 1 "${rc}" + +mkdir -p "${TMP}/empty" +out=$(resolve_o2pdpsuite_tag "${TMP}/empty" 2>/dev/null) ; rc=$? +check "directory with no dailies fails" 1 "${rc}" + +# A requested tag from a fork PR's body must not be able to escape the +# O2PDPSuite module directory via "..". +mkdir -p "${TMP}/O2" +touch "${TMP}/O2/something" +out=$(resolve_o2pdpsuite_tag "${D}" "../O2/something" 2>/dev/null) ; rc=$? +check "path traversal tag fails" 1 "${rc}" +check "path traversal tag prints nothing on stdout" "" "${out}" + +echo +if [[ "${FAILURES}" == "0" ]] ; then + echo "All resolve_tag tests passed" +else + echo "${FAILURES} test(s) failed" +fi +exit $(( FAILURES > 0 ? 1 : 0 )) diff --git a/test/tests/run_tests_selection.sh b/test/tests/run_tests_selection.sh new file mode 100755 index 000000000..a876b0a29 --- /dev/null +++ b/test/tests/run_tests_selection.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# Offline harness for run_tests.sh: stubs the sub-test scripts and checks that +# selection and exit-code aggregation behave. + +set -u +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ENTRY="${HERE}/../run_tests.sh" +FAILURES=0 + +check() +{ + local label=$1 expected=$2 actual=$3 + if [[ "${expected}" == "${actual}" ]] ; then + echo "ok - ${label}" + else + echo "FAIL - ${label}: expected '${expected}', got '${actual}'" + FAILURES=$((FAILURES + 1)) + fi +} + +# A stub directory with three scripts whose exit codes we control via env +# vars, and which each record the arguments they received to a sibling +# "*.args" file (one per line) so forwarding can be checked. +make_stubs() +{ + local dir=$1 + mkdir -p "${dir}" + for name in generator workflow relval ; do + cat > "${dir}/run_${name}_tests.sh" < '${dir}/run_${name}_tests.args' +exit \${STUB_${name^^}_RC:-0} +EOF + chmod +x "${dir}/run_${name}_tests.sh" + done +} + +run_entry() +{ + # usage: run_entry [args...]; prints nothing, sets OUT and RC + local dir=$1 ; shift + OUT=$(O2DPG_TEST_SUBTEST_DIR="${dir}" O2DPG_ROOT="${HERE}/../.." \ + O2DPG_TEST_REPO_DIR="${HERE}/../.." \ + bash "${ENTRY}" "$@" 2>&1) + RC=$? +} + +TMP=$(mktemp -d) +make_stubs "${TMP}/subtests" + +# 1. all three sub-tests run by default and a clean run exits 0 +run_entry "${TMP}/subtests" +check "default selection exits 0" 0 "${RC}" +check "default runs generator" 1 "$(grep -c 'STUB generator ran' <<< "${OUT}")" +check "default runs workflow" 1 "$(grep -c 'STUB workflow ran' <<< "${OUT}")" +check "default runs relval" 1 "$(grep -c 'STUB relval ran' <<< "${OUT}")" + +# 2. an explicit selection runs only what was asked for +run_entry "${TMP}/subtests" relval +check "explicit selection exits 0" 0 "${RC}" +check "explicit selection skips generator" 0 "$(grep -c 'STUB generator ran' <<< "${OUT}")" +check "explicit selection runs relval" 1 "$(grep -c 'STUB relval ran' <<< "${OUT}")" + +# 3. a failing sub-test makes the entrypoint fail, and the others still run +STUB_GENERATOR_RC=3 run_entry "${TMP}/subtests" +check "failing generator propagates" 3 "${RC}" +check "failure does not stop workflow" 1 "$(grep -c 'STUB workflow ran' <<< "${OUT}")" + +# 4. --fail-immediately stops after the first failure +STUB_GENERATOR_RC=3 run_entry "${TMP}/subtests" --fail-immediately +check "fail-immediately propagates" 3 "${RC}" +check "fail-immediately stops early" 0 "$(grep -c 'STUB relval ran' <<< "${OUT}")" + +# 5. a later failure is not masked by an earlier success +STUB_RELVAL_RC=4 run_entry "${TMP}/subtests" +check "late failure propagates" 4 "${RC}" + +# 6. flag forwarding is per sub-test: only "generator" accepts these flags +run_entry "${TMP}/subtests" --keep-artifacts +check "keep-artifacts reaches generator" \ + 1 "$(grep -cx -- '--keep-artifacts' "${TMP}/subtests/run_generator_tests.args")" +check "keep-artifacts does not reach workflow" \ + 0 "$(grep -cx -- '--keep-artifacts' "${TMP}/subtests/run_workflow_tests.args")" +check "keep-artifacts does not reach relval" \ + 0 "$(grep -cx -- '--keep-artifacts' "${TMP}/subtests/run_relval_tests.args")" + +echo +if [[ "${FAILURES}" == "0" ]] ; then + echo "All run_tests.sh selection tests passed" +else + echo "${FAILURES} test(s) failed" +fi +exit $(( FAILURES > 0 ? 1 : 0 ))