Skip to content

Commit dccbb21

Browse files
ChSonnabendChSonnabend
committed
Run O2 GPU CI tests
1 parent 48758af commit dccbb21

2 files changed

Lines changed: 113 additions & 0 deletions

File tree

‎Common/ML/test/onnxruntime-inference/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,6 @@ install(PROGRAMS
3030
run-onnxruntime-migraphx.sh
3131
run-onnxruntime-tensorrt.sh
3232
run-local-onnxruntime-inference-test.sh
33+
run-ci-onnxruntime-inference-test.sh
3334
DESTINATION bin)
3435
install(FILES net.onnx DESTINATION test/onnxruntime-inference)

‎Common/ML/test/onnxruntime-inference/run-ci-onnxruntime-inference-test.sh‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# CI entry point for the ONNX Runtime execution-provider inference test.
5+
#
6+
# This is meant to be called from an alidist recipe (O2-GPU-test) that has the
7+
# O2 dependency environment loaded (ONNXRuntime, gpu-system, CMake, ninja). It
8+
# derives the GPU backends to test the same way the other O2 GPU CI recipes do
9+
# (O2GPUCI_BACKENDS or gpu-features-available.sh), maps them onto the ONNX
10+
# Runtime execution providers ONNXRuntime was built with (ort-init.sh), and
11+
# fails if no GPU execution provider ends up being tested: a GPU CI check that
12+
# silently tests only the CPU provider is not a GPU CI check.
13+
#
14+
# usage: run-ci-onnxruntime-inference-test.sh [O2_SOURCEDIR]
15+
# O2_SOURCEDIR AliceO2 source tree. Defaults to the tree this script is in.
16+
#
17+
# Environment:
18+
# O2GPUCI_BACKENDS Comma/space separated list of backends to require
19+
# (CUDA, HIP). Defaults to what gpu-system detected.
20+
# BUILDDIR If set (as in alidist recipes), the test is built in
21+
# $BUILDDIR/onnxruntime-inference-test.
22+
# ONNXRUNTIME_INFERENCE_TEST_DEVICE_ID GPU device id, defaults to 0.
23+
24+
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
25+
O2_SOURCEDIR=${1:-$(cd "$SCRIPT_DIR/../../../.." && pwd)}
26+
TEST_DIR=$O2_SOURCEDIR/Common/ML/test/onnxruntime-inference
27+
TEST_SCRIPT=$TEST_DIR/run-local-onnxruntime-inference-test.sh
28+
29+
if [[ ! -f $TEST_SCRIPT ]]; then
30+
echo "onnxruntime-inference-ci: could not find test runner: $TEST_SCRIPT" >&2
31+
echo "Pass the AliceO2 source tree as the first argument." >&2
32+
exit 1
33+
fi
34+
35+
if [[ -z ${ONNXRUNTIME_ROOT:-} || ! -d $ONNXRUNTIME_ROOT/lib/cmake/onnxruntime ]]; then
36+
echo "onnxruntime-inference-ci: ONNXRUNTIME_ROOT is not set or does not contain lib/cmake/onnxruntime" >&2
37+
echo "ONNXRUNTIME_ROOT=${ONNXRUNTIME_ROOT:-}" >&2
38+
exit 1
39+
fi
40+
for TOOL in cmake ninja; do
41+
if ! command -v $TOOL > /dev/null; then
42+
echo "onnxruntime-inference-ci: $TOOL not found in PATH; add CMake and ninja to build_requires" >&2
43+
exit 1
44+
fi
45+
done
46+
47+
if [[ -f $ONNXRUNTIME_ROOT/etc/ort-init.sh ]]; then
48+
source "$ONNXRUNTIME_ROOT/etc/ort-init.sh"
49+
fi
50+
if [[ -n ${GPU_SYSTEM_ROOT:-} && -f $GPU_SYSTEM_ROOT/etc/gpu-features-available.sh ]]; then
51+
source "$GPU_SYSTEM_ROOT/etc/gpu-features-available.sh"
52+
fi
53+
54+
GPU_BACKENDS=()
55+
if [[ -n ${O2GPUCI_BACKENDS:-} ]]; then
56+
read -r -a GPU_BACKENDS <<< "${O2GPUCI_BACKENDS//,/ }"
57+
else
58+
[[ ${O2_GPU_CUDA_AVAILABLE:-0} == 1 ]] && GPU_BACKENDS+=(CUDA)
59+
[[ ${O2_GPU_ROCM_AVAILABLE:-0} == 1 ]] && GPU_BACKENDS+=(HIP)
60+
fi
61+
62+
if [[ ${#GPU_BACKENDS[@]} == 0 ]]; then
63+
echo "onnxruntime-inference-ci: no GPU backend selected or detected." >&2
64+
echo "Set O2GPUCI_BACKENDS='CUDA,HIP' in CI to require both production GPU backends." >&2
65+
exit 1
66+
fi
67+
68+
PROVIDERS=(cpu)
69+
for BACKEND in "${GPU_BACKENDS[@]}"; do
70+
case "${BACKEND^^}" in
71+
CUDA)
72+
if [[ ${ORT_CUDA_BUILD:-0} != 1 ]]; then
73+
echo "onnxruntime-inference-ci: CUDA backend requested but ONNXRuntime was built without the CUDA execution provider" >&2
74+
exit 1
75+
fi
76+
PROVIDERS+=(cuda)
77+
[[ ${ORT_TENSORRT_BUILD:-0} == 1 ]] && PROVIDERS+=(tensorrt)
78+
;;
79+
HIP|ROCM)
80+
if [[ ${ORT_MIGRAPHX_BUILD:-0} != 1 ]]; then
81+
echo "onnxruntime-inference-ci: HIP backend requested but ONNXRuntime was built without the MIGraphX execution provider" >&2
82+
exit 1
83+
fi
84+
PROVIDERS+=(migraphx)
85+
;;
86+
*)
87+
echo "onnxruntime-inference-ci: unsupported backend requested: $BACKEND" >&2
88+
exit 1
89+
;;
90+
esac
91+
done
92+
93+
if [[ ${#PROVIDERS[@]} == 1 ]]; then
94+
echo "onnxruntime-inference-ci: no ONNXRuntime GPU execution provider was selected." >&2
95+
echo "ORT_CUDA_BUILD=${ORT_CUDA_BUILD:-0}, ORT_TENSORRT_BUILD=${ORT_TENSORRT_BUILD:-0}, ORT_MIGRAPHX_BUILD=${ORT_MIGRAPHX_BUILD:-0}" >&2
96+
exit 1
97+
fi
98+
99+
BUILD_DIR=${BUILDDIR:-${TMPDIR:-/tmp}}/onnxruntime-inference-test
100+
PROVIDER_LIST=$(IFS=,; echo "${PROVIDERS[*]}")
101+
echo "onnxruntime-inference-ci: backends: ${GPU_BACKENDS[*]}; providers: $PROVIDER_LIST"
102+
103+
EXTRA_ARGS=()
104+
[[ -n ${ONNXRUNTIME_INFERENCE_TEST_DEVICE_ID:-} ]] && EXTRA_ARGS+=(--device-id "$ONNXRUNTIME_INFERENCE_TEST_DEVICE_ID")
105+
106+
# The local runner must never fall back to alienv bootstrapping in CI.
107+
export ONNXRUNTIME_INFERENCE_TEST_BOOTSTRAPPED=1
108+
"$TEST_SCRIPT" --model "$TEST_DIR/net.onnx" \
109+
--build-dir "$BUILD_DIR" \
110+
--providers "$PROVIDER_LIST" \
111+
${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}
112+
rm -Rf "$BUILD_DIR"

0 commit comments

Comments
 (0)