Fix profiler and TAU harness checks

HIP/jacobi/Makefile

@@ -27,7 +27,13 @@ ifneq ($(USE_CRAYPE),)
   MPICC=CC
 endif
 
-IS_OMPI=$(shell which ompi_info 2>/dev/null)
+# Ask the wrapper we are about to invoke. `which ompi_info` reports that some
+# OpenMPI is installed, which is not the question: with MPICH wrappers in front
+# (any MPICH-built module can pull them in) this branch asked MPICH's mpic++ for
+# --showme:compile, got the flag forwarded to the compiler as an unknown
+# argument, and left both flag sets empty -- so the link had no MPI library and
+# produced a page of undefined symbols.
+IS_OMPI=$(shell ${MPICC} --showme:version >/dev/null 2>&1 && echo openmpi)
 # MPICH detection. The Cray cc/CC wrapper prints "CrayPE is loaded" (matched by
 # 'Cray' below). The standalone mpich-wrappers MPICH on a Cray, however, reports
 # the underlying compiler from `--version` (e.g. "AMD clang"), so grepping the
@@ -58,6 +64,14 @@ else
     $(error Unknown MPI version! Currently can detect mpich or openmpi)
 endif
 
+# MPILD is hipcc, not the MPI wrapper, so the MPI library only reaches the link
+# line through MPILDFLAGS. Empty flags therefore produce a page of undefined MPI
+# symbols that names no cause. Fail here instead, while the reason is still
+# visible.
+ifeq ($(strip $(MPILDFLAGS)),)
+    $(error MPI detected but $(strip ${MPICC}) returned no link flags. Two MPIs on PATH is the usual cause: check that mpic++ and any MPI-linked module you loaded are the same MPI)
+endif
+
 # Flags
 CFLAGS=-O3 -g -ggdb -fPIC -std=c++11 -march=native -Wall
 CFLAGS+=-I$(ROCM_PATH)/roctracer/include -I${ROCM_PATH}/include/roctracer

MLExamples/PyTorch_Profiling/rocm-compute-profiler/single_process.sh

@@ -7,6 +7,39 @@ PROFILER_TOP_DIR="$(dirname "$(dirname "$(readlink -fm "$0")")")"
 # Call the software set up script:
 source ${PROFILER_TOP_DIR}/setup.sh
 
+# Dependency precondition. `analyze` enforces the exact pins in its own
+# requirements.txt and this run ends in `analyze`, so verify them before
+# spending around 25 minutes profiling. Fail fast rather than time out.
+_rpc_bin="$(command -v rocprof-compute 2>/dev/null)"
+if [ -n "${_rpc_bin}" ]; then
+  _rpc_req="$(dirname "$(dirname "${_rpc_bin}")")/libexec/rocprofiler-compute/requirements.txt"
+  if [ -f "${_rpc_req}" ] && ! python3 - "${_rpc_req}" <<'PINS'
+import sys
+from importlib.metadata import version, PackageNotFoundError
+bad = []
+for raw in open(sys.argv[1]):
+    raw = raw.split('#')[0].strip()
+    if not raw or '==' not in raw:
+        continue
+    name, want = raw.split('==', 1)
+    try:
+        have = version(name)
+    except PackageNotFoundError:
+        bad.append('%s: absent (needs %s)' % (name, want)); continue
+    if have != want:
+        bad.append('%s: %s installed, needs %s' % (name, have, want))
+for b in bad:
+    print('  ' + b)
+sys.exit(1 if bad else 0)
+PINS
+  then
+    echo "rocprof-compute dependency pins are unsatisfied under $(python3 -V 2>&1)."
+    echo "'analyze' cannot run, so skipping the profile run instead of timing out."
+    echo "See ${_rpc_req} for the required versions."
+    exit 1
+  fi
+fi
+
 export NPROCS=1
 
 pushd ${PROFILER_TOP_DIR}

tests/CMakeLists.txt

@@ -1696,11 +1696,11 @@ set_property(TEST Rocprof-sys_ROCm_Run_Check PROPERTY PASS_REGULAR_EXPRESSION "T
 set_property(TEST Rocprof-sys_ROCm_Run_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider")
 
 add_test(NAME Rocprof-sys_ROCm_Stream_Overlap COMMAND ../rocprof-sys_stream_overlap.sh )
-set_property(TEST Rocprof-sys_ROCm_Stream_Overlap PROPERTY PASS_REGULAR_EXPRESSION "proto")
+set_property(TEST Rocprof-sys_ROCm_Stream_Overlap PROPERTY PASS_REGULAR_EXPRESSION "ROCPROFSYS_STREAM_OVERLAP_RESULT: PASS")
 set_property(TEST Rocprof-sys_ROCm_Stream_Overlap PROPERTY SKIP_REGULAR_EXPRESSION "module spider")
 
 add_test(NAME Rocprof-compute_ROCm_Roofline_Check COMMAND ../rocprof-compute_roofline_check.sh)
-set_property(TEST Rocprof-compute_ROCm_Roofline_Check PROPERTY PASS_REGULAR_EXPRESSION "Empirical Roofline .* saved!")
+set_property(TEST Rocprof-compute_ROCm_Roofline_Check PROPERTY PASS_REGULAR_EXPRESSION "ROOFLINE_RESULT: PASS")
 set_property(TEST Rocprof-compute_ROCm_Roofline_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider" "disabled on MI300")
 set_tests_properties(Rocprof-compute_ROCm_Roofline_Check PROPERTIES TIMEOUT 600)
 
@@ -1756,19 +1756,19 @@ set_property(TEST JAX_Mnist PROPERTY SKIP_REGULAR_EXPRESSION "module spider" "Un
 # TAU Tests
 
 add_test(NAME TAU_Trace_Check COMMAND ../tau_exec_check.sh --tau-trace)
-set_property(TEST TAU_Trace_Check PROPERTY PASS_REGULAR_EXPRESSION "tautrace.0")
+set_property(TEST TAU_Trace_Check PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: trace artifact PRESENT")
 set_property(TEST TAU_Trace_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'")
 
 add_test(NAME TAU_Profile_Check COMMAND ../tau_exec_check.sh --tau-profile)
-set_property(TEST TAU_Profile_Check PROPERTY PASS_REGULAR_EXPRESSION "profile.0")
+set_property(TEST TAU_Profile_Check PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: profile artifact PRESENT")
 set_property(TEST TAU_Profile_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'")
 
 add_test(NAME TAU_Check_HIP_Profile COMMAND ../tau_exec_check.sh --tau-profile)
-set_property(TEST TAU_Check_HIP_Profile PROPERTY PASS_REGULAR_EXPRESSION "hipMemcpy")
+set_property(TEST TAU_Check_HIP_Profile PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: HIP routines PRESENT in profile")
 set_property(TEST TAU_Check_HIP_Profile PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'")
 
 add_test(NAME TAU_Check_MPI_Profile COMMAND ../tau_exec_check.sh --tau-profile)
-set_property(TEST TAU_Check_MPI_Profile PROPERTY PASS_REGULAR_EXPRESSION "MPI_Allreduce()")
+set_property(TEST TAU_Check_MPI_Profile PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: MPI routines PRESENT in profile")
 set_property(TEST TAU_Check_MPI_Profile PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'")
 set_property(TEST TAU_Check_MPI_Profile PROPERTY TIMEOUT 600)
 

tests/rocprof-compute_analyze_check.sh

@@ -22,5 +22,13 @@ cmake ..
 make
 
 export HSA_XNACK=1
+# rocprof-compute needs Python >= 3.10; native_tool_finder.py uses PEP 604
+# unions. Fail here, naming the reason, rather than later with a SyntaxError
+# from inside the tool that names no cause.
+if ! python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3,10) else 1)' 2>/dev/null; then
+  echo "ERROR: rocprof-compute needs Python >= 3.10, but python3 is $(python3 -V 2>&1)."
+  echo "ERROR: load a newer Python before running this test."
+  exit 1
+fi
 rocprof-compute profile -n v1 --no-roof -- ./saxpy
 rocprof-compute analyze -p workloads/v1/* --block 7.1.0 7.1.1 7.1.2 7.1.0: Grid size 7.1.1: Workgroup size 7.1.2: Total Wavefronts

tests/rocprof-compute_profile_check.sh

@@ -23,4 +23,12 @@ cmake ..
 make
 
 export HSA_XNACK=1
+# rocprof-compute needs Python >= 3.10; native_tool_finder.py uses PEP 604
+# unions. Fail here, naming the reason, rather than later with a SyntaxError
+# from inside the tool that names no cause.
+if ! python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3,10) else 1)' 2>/dev/null; then
+  echo "ERROR: rocprof-compute needs Python >= 3.10, but python3 is $(python3 -V 2>&1)."
+  echo "ERROR: load a newer Python before running this test."
+  exit 1
+fi
 rocprof-compute profile -n v1 --no-roof -- ./saxpy

tests/rocprof-compute_roofline_check.sh

@@ -30,4 +30,21 @@ cmake ..
 make
 
 export HSA_XNACK=1
+# rocprof-compute needs Python >= 3.10; native_tool_finder.py uses PEP 604
+# unions. Fail here, naming the reason, rather than later with a SyntaxError
+# from inside the tool that names no cause.
+if ! python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3,10) else 1)' 2>/dev/null; then
+  echo "ERROR: rocprof-compute needs Python >= 3.10, but python3 is $(python3 -V 2>&1)."
+  echo "ERROR: load a newer Python before running this test."
+  exit 1
+fi
 rocprof-compute profile -n rooflines_PDF --roof-only  -- ./saxpy
+
+# Assert the roofline artifact exists rather than matching the tool's log
+# wording, which changed when omniperf became rocprofiler-compute.
+_roofline_csv="$(find . -name roofline.csv -size +0 2>/dev/null | head -1)"
+if [ -n "${_roofline_csv}" ]; then
+  echo "ROOFLINE_RESULT: PASS roofline data at ${_roofline_csv}"
+else
+  echo "ROOFLINE_RESULT: FAIL no non-empty roofline.csv produced"
+fi

tests/rocprof-sys_stream_overlap.sh

@@ -126,9 +126,28 @@ ${TOOL_COMMAND}-avail -G $PWD/.configure.cfg
 export ${TOOL_CONFIG}_CONFIG_FILE=$PWD/.configure.cfg
 ${TOOL_COMMAND}-instrument -o compute_comm_overlap.inst -- compute_comm_overlap
 ${TOOL_COMMAND}-run -- ./compute_comm_overlap.inst 2
-cd ${TOOL_OUTPUT}-compute_comm_overlap.inst-output/
-ls *
-
+# Assert a real trace artifact. The tool writes
+# ${TOOL_NAME}-compute_comm_overlap.inst-output (rocprofiler-systems-... on
+# ROCm > 6.2.9, omnitrace-... before it), NOT ${TOOL_OUTPUT}-...  Glob for the
+# suffix so both eras work, and require a real .proto inside it.
+assert_proto_output() {
+  local outdir n
+  outdir="$(ls -d ./*-compute_comm_overlap.inst-output 2>/dev/null | head -1)"
+  if [[ -z "${outdir}" || ! -d "${outdir}" ]]; then
+    echo "ROCPROFSYS_STREAM_OVERLAP_RESULT: FAIL no *-compute_comm_overlap.inst-output directory produced"
+    return 1
+  fi
+  # The tool nests a timestamped subdirectory, so recurse.
+  n="$(find "${outdir}" -type f -name '*.proto' | wc -l)"
+  find "${outdir}" -type f | sed 's/^/    /' | head -20
+  if [[ "${n}" -gt 0 ]]; then
+    echo "ROCPROFSYS_STREAM_OVERLAP_RESULT: PASS ${n} proto file(s) under ${outdir}"
+  else
+    echo "ROCPROFSYS_STREAM_OVERLAP_RESULT: FAIL 0 proto files under ${outdir}"
+    return 1
+  fi
+}
+assert_proto_output
 cd ..
 rm -rf ${BUILD_DIR}
 

tests/tau_exec_check.sh

@@ -84,10 +84,23 @@ export TAU_PROFILE=${TAU_PROFILE}
 export TAU_TRACE=${TAU_TRACE}
 
 WORKDIR=$(mktemp -d -p ${SRCDIR} build_XXXXXX)
+# Install the cleanup trap before the build gate below, which can exit early.
+trap 'cd "${SRCDIR}" && rm -rf "${WORKDIR}"' EXIT
 cp ${SRCDIR}/*.hip ${SRCDIR}/*.hpp ${SRCDIR}/*.h ${SRCDIR}/Makefile ${SRCDIR}/input.txt ${WORKDIR}/
 cd ${WORKDIR}
 
-make
+# A failed build has to end the test. Without this gate make's status was
+# ignored, tau_exec profiled a binary that was never produced, and the script
+# still exited 0 because cleanup was the last command -- leaving the whole
+# verdict to the pass regex.
+if ! make; then
+   echo "TAU_CHECK: build FAILED, so nothing was profiled"
+   exit 1
+fi
+if [ ! -x ./Jacobi_hip ]; then
+   echo "TAU_CHECK: build reported success but ./Jacobi_hip is missing"
+   exit 1
+fi
 
 ROCM_VERSION=`cat ${ROCM_PATH}/.info/version | head -1 | cut -f1 -d'-' `
 
@@ -117,10 +130,55 @@ elif [ -n "${MPI_BINDIR}" ] && [ -x "${MPI_BINDIR}/mpiexec" ]; then
 else
    MPI_LAUNCH="srun -n 2"
 fi
-if command -v ompi_info >/dev/null 2>&1; then
+# Ask the wrapper we are about to invoke, not the PATH. `command -v ompi_info`
+# answers "is some OpenMPI installed", which is a different question. An
+# MPICH-built TAU module makes the two disagree: loading it pulls in
+# mpich-wrappers, so mpicc/mpic++/mpiexec become MPICH's while ompi_info still
+# reports the OpenMPI that is no longer in front. --showme:version succeeds only
+# on OpenMPI and -compile_info only on MPICH, so each wrapper answers for itself.
+MPI_FAMILY=unknown
+if [ -n "${MPI_BINDIR}" ]; then
+   if "${MPI_BINDIR}/mpicc" --showme:version >/dev/null 2>&1; then
+      MPI_FAMILY=openmpi
+   elif "${MPI_BINDIR}/mpicc" -compile_info >/dev/null 2>&1; then
+      MPI_FAMILY=mpich
+   fi
+fi
+echo "TAU_CHECK: launching with ${MPI_FAMILY} wrappers from ${MPI_BINDIR:-<none, using srun>}"
+if [ "${MPI_FAMILY}" = "openmpi" ]; then
    MPI_LAUNCH="${MPI_LAUNCH} --oversubscribe"
 fi
 
+# Keep both ranks on the node that owns the build directory. Jacobi is compiled
+# into a mktemp directory inside this checkout, so if the checkout lives on
+# node-local storage rather than a shared filesystem, a multi-node allocation
+# lets the launcher place rank 1 on a node where that path does not exist. The
+# run then dies before the test body starts:
+#
+#   error: couldn't chdir to `/tmp/.../HIP/jacobi/build_ewLCPv': No such file or
+#   directory: going to /tmp instead
+#   [proxy:1@node2] launch_procs: unable to change wdir to /tmp/.../build_ewLCPv
+#
+# The test wants two ranks, not two nodes -- Jacobi's "-g 2 1" topology on two
+# GPUs of one node -- so pinning costs no coverage. Hydra takes -hosts; OpenMPI
+# takes --host with a slot count. Measured on two nodes: bare mpirun aborts as
+# above, both pinned forms put rank 0 and rank 1 on the same node.
+#
+# The srun fallback is left alone deliberately: it is reached only for a bare
+# MPICH with no co-located launcher, and the obvious addition there
+# (--nodes=1 --nodelist=...) needs the step's GRES respecified or Slurm rejects
+# the step outright, so a blind flag would trade a rare failure for a common one.
+MPI_HOST_LOCAL="$(hostname -s)"
+case "${MPI_LAUNCH}" in
+   srun*) ;;
+   *) case "${MPI_FAMILY}" in
+         openmpi) MPI_LAUNCH="${MPI_LAUNCH} --host ${MPI_HOST_LOCAL}:2" ;;
+         mpich)   MPI_LAUNCH="${MPI_LAUNCH} -hosts ${MPI_HOST_LOCAL}" ;;
+         *) echo "TAU_CHECK: unknown MPI family, not pinning ranks to ${MPI_HOST_LOCAL}" ;;
+      esac ;;
+esac
+echo "TAU_CHECK: launcher is ${MPI_LAUNCH}"
+
 # Use a 1024x1024 local mesh so the TAU trace buffer fits in GPU memory
 # (default 4096x4096 caused "HIP failure: 'out of memory'" during trace finalization).
 if [[ "${result}" ]]; then
@@ -130,8 +188,43 @@ else
 fi
 
 ls
-pprof
+pprof 2>&1 | tee pprof.out
+
+# Assert the artifacts. Matching the tool's own vocabulary is what made these
+# tests unable to fail: "profile.0" is a substring of "Could not open
+# profile.0.0.0". Assert the artifact and the profiled routines instead, in
+# strings the failure paths cannot produce, and let CMake match these.
+tau_have() { ls $1 >/dev/null 2>&1; }
+tau_status=0
+
+if [ "${TAU_PROFILE}" = "1" ]; then
+   if tau_have 'profile.*'; then
+      echo "TAU_CHECK: profile artifact PRESENT"
+   else
+      echo "TAU_CHECK: profile artifact absent"
+      tau_status=1
+   fi
+   # Reported but not fatal on their own: each is a different test's subject, so
+   # the per-test expression decides rather than failing all of them together.
+   if grep -q 'MPI_Allreduce' pprof.out; then
+      echo "TAU_CHECK: MPI routines PRESENT in profile"
+   else
+      echo "TAU_CHECK: MPI routines absent from profile"
+   fi
+   if grep -q 'hipMemcpy' pprof.out; then
+      echo "TAU_CHECK: HIP routines PRESENT in profile"
+   else
+      echo "TAU_CHECK: HIP routines absent from profile"
+   fi
+fi
 
-cd ..
-rm -rf ${WORKDIR}
+if [ "${TAU_TRACE}" = "1" ]; then
+   if tau_have 'tautrace.*'; then
+      echo "TAU_CHECK: trace artifact PRESENT"
+   else
+      echo "TAU_CHECK: trace artifact absent"
+      tau_status=1
+   fi
+fi
 
+exit ${tau_status}

tests/tau_rccl_pc_check.sh

@@ -113,7 +113,21 @@ elif [ -n "${MPI_BINDIR}" ] && [ -x "${MPI_BINDIR}/mpiexec" ]; then
 else
    MPI_LAUNCH="srun -n 2"
 fi
-if command -v ompi_info >/dev/null 2>&1; then
+# Ask the wrapper we are about to invoke; see tau_exec_check.sh for the detail.
+# Testing whether ompi_info exists added OpenMPI's --oversubscribe to MPICH's
+# hydra mpiexec, which rejects it. Note this does not by itself make the test
+# pass where TAU and the prebuilt rccl-tests binary come from different MPIs:
+# an MPICH-built TAU cannot instrument an OpenMPI-linked benchmark.
+MPI_FAMILY=unknown
+if [ -n "${MPI_BINDIR}" ]; then
+   if "${MPI_BINDIR}/mpicc" --showme:version >/dev/null 2>&1; then
+      MPI_FAMILY=openmpi
+   elif "${MPI_BINDIR}/mpicc" -compile_info >/dev/null 2>&1; then
+      MPI_FAMILY=mpich
+   fi
+fi
+echo "MPI family of ${MPI_BINDIR:-<none, using srun>}: ${MPI_FAMILY}"
+if [ "${MPI_FAMILY}" = "openmpi" ]; then
    MPI_LAUNCH="${MPI_LAUNCH} --oversubscribe"
 fi