Add GPU-initiated and Host-initiated SDMA executors via Anvil

CHANGELOG.md

@@ -8,6 +8,11 @@ Documentation for TransferBench is available at
 - Added support for SWEEP_MIN_POW2 and SWEEP_MAX_POW2 to set sweep bounds when bytes to transfer is 0
 - Adding support for Tensor Data Mover (TDM)-based executor [T] on supported hardware.  This provides
   an alternative data movement mechanism which utilizes async loads to shared memory / from shared memory
+- Adding new GPU-initiated SDMA executor "GMA" [S] (AMD only, opt-in via `-DENABLE_ANVIL_EXEC=ON`), which
+  drives KFD SDMA queues directly from a GPU kernel using the bundled `anvil` KFD/SDMA queue library
+- Adding new host-initiated SDMA executor "HMA" [H] (AMD only), a CPU-driven variant of the anvil SDMA
+  path where the host builds packets, rings the doorbell, and polls for completion
+- Extended the AllToAll preset `USE_DMA_EXEC` to select the executor: 0=GFX, 1=DMA, 2=BMA, 3=GMA, 4=HMA
 ### Modified
 - CPU NUMA nodes with 0 cores will now be hidden.  To re-enable, set TB_SHOW_ALL_NUMA=1
 - Switching to use of persistent threadpools to cut-down on thread creation overheads

CMakeLists.txt

@@ -203,6 +203,7 @@ option(ENABLE_MPI_COMM             "Enable MPI Communicator support"
 option(ENABLE_AMD_SMI              "Enable AMD-SMI pod membership queries"        OFF)
 option(ENABLE_POD_COMM             "Enable pod communication"                     OFF)
 option(BUILD_RELOCATABLE_PACKAGE   "Build with RVS-style relocatable RPATH and amdrocm<MAJOR>-transferbench package naming" OFF)
+option(ENABLE_ANVIL_EXEC           "Enable GPU-initiated SDMA executor via anvil (AMD MI300X only)" OFF)
 
 include(cmake/Dependencies.cmake) # rocm-cmake, rocm_local_targets, rocm_check_target_ids
 
@@ -440,6 +441,46 @@ else()
   endif()
 endif()
 
+## Check for anvil SDMA executor support
+if(NOT ENABLE_ANVIL_EXEC)
+  message(STATUS "For CMake builds, anvil GPU-initiated SDMA executor requires -DENABLE_ANVIL_EXEC=ON")
+  message(STATUS "- Disabling anvil executor support")
+else()
+  message(STATUS "Attempting to build with anvil GPU-initiated SDMA executor")
+
+  find_library(HSAKMT_LIBRARY hsakmt PATHS ${ROCM_PATH}/lib ${ROCM_PATH}/lib64 NO_DEFAULT_PATH)
+  find_path(HSAKMT_INCLUDE_DIR hsakmt/hsakmt.h PATHS ${ROCM_PATH}/include NO_DEFAULT_PATH)
+
+  if(HSAKMT_LIBRARY AND HSAKMT_INCLUDE_DIR)
+    # hsakmt is typically a static archive; drm_amdgpu/drm are transitive deps
+    if(HSAKMT_LIBRARY MATCHES "\\.a$")
+      add_library(hsakmt STATIC IMPORTED)
+    else()
+      add_library(hsakmt SHARED IMPORTED)
+    endif()
+    find_library(DRM_AMDGPU_LIBRARY drm_amdgpu)
+    find_library(DRM_LIBRARY drm)
+    set_target_properties(hsakmt PROPERTIES
+      IMPORTED_LOCATION "${HSAKMT_LIBRARY}"
+      INTERFACE_INCLUDE_DIRECTORIES "${HSAKMT_INCLUDE_DIR}"
+      INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${HSAKMT_INCLUDE_DIR}"
+      INTERFACE_LINK_LIBRARIES "${DRM_AMDGPU_LIBRARY};${DRM_LIBRARY}")
+    set(ANVIL_EXEC_FOUND 1)
+    message(STATUS "- Building with anvil GPU-initiated SDMA executor support")
+    message(STATUS "  - hsakmt: ${HSAKMT_LIBRARY}")
+    message(STATUS "  - drm_amdgpu: ${DRM_AMDGPU_LIBRARY}")
+    message(STATUS "  - drm: ${DRM_LIBRARY}")
+  else()
+    if(NOT HSAKMT_LIBRARY)
+      message(WARNING "- libhsakmt not found under ${ROCM_PATH}/lib or ${ROCM_PATH}/lib64")
+    endif()
+    if(NOT HSAKMT_INCLUDE_DIR)
+      message(WARNING "- hsakmt/hsakmt.h not found under ${ROCM_PATH}/include")
+    endif()
+    message(WARNING "- Building without anvil executor support")
+  endif()
+endif()
+
 set(PACKAGE_NAME TB)
 set(LIBRARY_NAME TransferBench)
 
@@ -472,6 +513,23 @@ endif()
 if(POD_COMM_FOUND)
   target_compile_definitions(TransferBench PRIVATE POD_COMM_ENABLED)
 endif()
+if(ANVIL_EXEC_FOUND)
+  target_sources(TransferBench PRIVATE src/anvil/anvil.cpp)
+  target_include_directories(TransferBench PRIVATE src/anvil)
+  target_compile_definitions(TransferBench PRIVATE ANVIL_EXEC_ENABLED)
+  target_link_libraries(TransferBench PRIVATE hsakmt)
+  # XIO_SDMA_OSS7: fused COPY_LINEAR_WAIT_SIGNAL_MI4 packet. ABI-portable (structs
+  # + host code build on any arch; device code is arch-gated to gfx1250/gfx950 via
+  # XIO_SDMA_OSS7_ENABLED and selected at runtime only on gfx1250), so it is safe
+  # for all-arch builds. Set -DENABLE_XIO_SDMA_OSS7=OFF to force the separate path.
+  option(ENABLE_XIO_SDMA_OSS7 "Enable fused MI4 SDMA packets (gfx1250 runtime)" ON)
+  if(ENABLE_XIO_SDMA_OSS7)
+    target_compile_definitions(TransferBench PRIVATE XIO_SDMA_OSS7=1)
+    message(STATUS "- Building with XIO_SDMA_OSS7 (fused MI4 SDMA packets; device code gated to gfx1250/gfx950)")
+  else()
+    message(STATUS "- Building without XIO_SDMA_OSS7 (separate COPY_LINEAR + ATOMIC path)")
+  endif()
+endif()
 
 target_compile_definitions(TransferBench PRIVATE
   TB_GIT_BRANCH="${TB_GIT_BRANCH}"

Makefile

@@ -19,11 +19,12 @@ NVCC      ?= $(CUDA_PATH)/bin/nvcc
 DEBUG     ?= 0
 
 # Optional features (set to 0 to disable, 1 to enable)
-# DISABLE_MPI_COMM: Disable MPI communicator support                      (default: 0)
-# DISABLE_AMD_SMI:  Disable AMD-SMI pod membership checking support       (default: 0)
-# DISABLE_NVML:     Disable NVML pod membership detection for CUDA builds (default: 0)
-# DISABLE_POD_COMM: Disable pod communication support                     (default: 0)
-# DISABLE_CUMEM:    Disable CUDA driver API (also disables pod on CUDA)   (default: 0)
+# DISABLE_MPI_COMM:  Disable MPI communicator support                      (default: 0)
+# DISABLE_AMD_SMI:   Disable AMD-SMI pod membership checking support       (default: 0)
+# DISABLE_NVML:      Disable NVML pod membership detection for CUDA builds (default: 0)
+# DISABLE_POD_COMM:  Disable pod communication support                     (default: 0)
+# DISABLE_CUMEM:     Disable CUDA driver API (also disables pod on CUDA)   (default: 0)
+# ENABLE_ANVIL_EXEC: Enable GPU-initiated SDMA executor (AMD KFD, ROCm only) (default: 0)
 
 # ROCm device libraries can live in different locations depending on packaging.
 # hipcc/clang needs to find the amdgcn bitcode directory at link time.
@@ -266,14 +267,75 @@ _TB_DIR       := $(dir $(abspath $(firstword $(MAKEFILE_LIST))))
 TB_GIT_BRANCH := $(shell git -C "$(_TB_DIR)" rev-parse --abbrev-ref HEAD 2>/dev/null || sed -n '1p' "$(_TB_DIR)GIT_VERSION" 2>/dev/null || echo unknown)
 TB_GIT_COMMIT := $(shell git -C "$(_TB_DIR)" rev-parse --short HEAD 2>/dev/null || sed -n '2p' "$(_TB_DIR)GIT_VERSION" 2>/dev/null || echo unknown)
 COMMON_FLAGS  += -DTB_GIT_BRANCH='"$(TB_GIT_BRANCH)"' -DTB_GIT_COMMIT='"$(TB_GIT_COMMIT)"'
+
+  ANVIL_ENABLED = 0
+  # Compile with GPU-initiated SDMA executor (anvil/KFD) if
+  # 1) ENABLE_ANVIL_EXEC is set to 1
+  # 2) hsakmt/hsakmt.h is found (KFD user-space library header)
+  # 3) libhsakmt is found (static or shared)
+  # Note: disabled by default; requires AMD ROCm KFD and is AMD-only
+  ENABLE_ANVIL_EXEC ?= 0
+  ifeq ($(ENABLE_ANVIL_EXEC), 1)
+    ifeq ($(MAKECMDGOALS),TransferBenchCuda)
+      $(info - Anvil executor not supported for CUDA builds; ignoring ENABLE_ANVIL_EXEC=1)
+    else
+      $(info Attempting to build with Anvil GPU-initiated SDMA executor support)
+      HSAKMT_INC := $(firstword $(wildcard \
+        $(ROCM_PATH)/include/hsakmt/hsakmt.h \
+        /usr/include/hsakmt/hsakmt.h))
+      HSAKMT_LIB := $(firstword $(wildcard \
+        $(ROCM_PATH)/lib/libhsakmt.a \
+        $(ROCM_PATH)/lib64/libhsakmt.a \
+        $(ROCM_PATH)/lib/libhsakmt.so \
+        /usr/lib/x86_64-linux-gnu/libhsakmt.so))
+      ifeq ($(HSAKMT_INC),)
+        $(info - hsakmt/hsakmt.h not found; cannot build Anvil executor)
+        $(info - Install libhsakmt-dev or ensure ROCM_PATH is set correctly)
+      else ifeq ($(HSAKMT_LIB),)
+        $(info - libhsakmt not found; cannot build Anvil executor)
+        $(info - Install libhsakmt or ensure ROCM_PATH is set correctly)
+      else
+        ANVIL_ENABLED = 1
+        COMMON_FLAGS += -DANVIL_EXEC_ENABLED -I./src/anvil
+        # XIO_SDMA_OSS7: fused COPY_LINEAR_WAIT_SIGNAL_MI4 packet. ABI-portable
+        # (device code is arch-gated to gfx1250/gfx950 via XIO_SDMA_OSS7_ENABLED and
+        # selected at runtime only on gfx1250). Set ENABLE_XIO_SDMA_OSS7=0 to force
+        # the separate COPY_LINEAR + ATOMIC path.
+        ENABLE_XIO_SDMA_OSS7 ?= 1
+        ifeq ($(ENABLE_XIO_SDMA_OSS7), 1)
+          COMMON_FLAGS += -DXIO_SDMA_OSS7=1
+          $(info - Building with XIO_SDMA_OSS7 (fused MI4 SDMA packets; device code gated to gfx1250/gfx950))
+        else
+          $(info - Building without XIO_SDMA_OSS7 (separate COPY_LINEAR + ATOMIC path))
+        endif
+        # Link hsakmt; use -Wl, to pass the static archive directly to the
+        # linker — without it, -x hip causes the compiler to parse the .a
+        # as source and emit "expected unqualified-id" / UTF-8 errors.
+        ifeq ($(suffix $(HSAKMT_LIB)),.a)
+          LDFLAGS += -Wl,$(HSAKMT_LIB) -ldrm_amdgpu -ldrm
+        else
+          LDFLAGS += -lhsakmt
+        endif
+        $(info - Building with Anvil GPU-initiated SDMA executor. Can set ENABLE_ANVIL_EXEC=0 to disable)
+      endif
+    endif
+    ifeq ($(ANVIL_ENABLED), 0)
+      $(info - Building without Anvil GPU-initiated SDMA executor support)
+    endif
+  endif
 endif
 
 .PHONY : all clean
 
 all: TransferBench
 
-TransferBench: ./src/client/Client.cpp $(shell find -regex ".*\.\hpp")
-	$(CXX) $(CXXFLAGS) $(HIPFLAGS) $(COMMON_FLAGS) $< -o $@ $(HIPLDFLAGS) $(LDFLAGS)
+ANVIL_SRCS =
+ifeq ($(ANVIL_ENABLED), 1)
+  ANVIL_SRCS = ./src/anvil/anvil.cpp
+endif
+
+TransferBench: ./src/client/Client.cpp $(ANVIL_SRCS) $(shell find -regex ".*\.\hpp")
+	$(CXX) $(CXXFLAGS) $(HIPFLAGS) $(COMMON_FLAGS) ./src/client/Client.cpp $(ANVIL_SRCS) -o $@ $(HIPLDFLAGS) $(LDFLAGS)
 
 TransferBenchCuda: ./src/client/Client.cpp $(shell find -regex ".*\.\hpp")
 	$(NVCC) $(NVFLAGS) $(COMMON_FLAGS) $< -o $@ $(LDFLAGS)